plotjuggler falling back to qlog (#20105)

* fallback to qlogs

* minor fixes

* removed extra line

* help on no args

* Update tools/plotjuggler/juggle.py

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>
old-commit-hash: d4feecd803

tools/lib/route.py

@@ -11,6 +11,7 @@ SEGMENT_NAME_RE = r'[a-z0-9]{16}[|_][0-9]{4}-[0-9]{2}-[0-9]{2}--[0-9]{2}-[0-9]{2
 EXPLORER_FILE_RE = r'^({})--([a-z]+\.[a-z0-9]+)$'.format(SEGMENT_NAME_RE)
 OP_SEGMENT_DIR_RE = r'^({})$'.format(SEGMENT_NAME_RE)
 
+QLOG_FILENAMES = ['qlog.bz2']
 LOG_FILENAMES = ['rlog.bz2', 'raw_log.bz2']
 CAMERA_FILENAMES = ['fcamera.hevc', 'video.hevc']
 
@@ -21,20 +22,23 @@ class Route(object):
       self._segments = self._get_segments_local(data_dir)
     else:
       self._segments = self._get_segments_remote()
+    self.max_seg_number = self._segments[-1].canonical_name.segment_num
 
   @property
   def segments(self):
     return self._segments
 
   def log_paths(self):
-    max_seg_number = self._segments[-1].canonical_name.segment_num
     log_path_by_seg_num = {s.canonical_name.segment_num: s.log_path for s in self._segments}
-    return [log_path_by_seg_num.get(i, None) for i in range(max_seg_number+1)]
+    return [log_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
+
+  def qlog_paths(self):
+    qlog_path_by_seg_num = {s.canonical_name.segment_num: s.qlog_path for s in self._segments}
+    return [qlog_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
 
   def camera_paths(self):
-    max_seg_number = self._segments[-1].canonical_name.segment_num
     camera_path_by_seg_num = {s.canonical_name.segment_num: s.camera_path for s in self._segments}
-    return [camera_path_by_seg_num.get(i, None) for i in range(max_seg_number+1)]
+    return [camera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
 
   def _get_segments_remote(self):
     api = CommaApi(get_token())
@@ -48,12 +52,14 @@ class Route(object):
         segments[segment_name] = RouteSegment(
           segment_name,
           url if fn in LOG_FILENAMES else segments[segment_name].log_path,
+          url if fn in QLOG_FILENAMES else segments[segment_name].qlog_path,
           url if fn in CAMERA_FILENAMES else segments[segment_name].camera_path
         )
       else:
         segments[segment_name] = RouteSegment(
           segment_name,
           url if fn in LOG_FILENAMES else None,
+          url if fn in QLOG_FILENAMES else None,
           url if fn in CAMERA_FILENAMES else None
         )
 
@@ -88,26 +94,33 @@ class Route(object):
 
     segments = []
     for segment, files in segment_files.items():
+
       try:
         log_path = next(path for path, filename in files if filename in LOG_FILENAMES)
       except StopIteration:
         log_path = None
 
+      try:
+        qlog_path = next(path for path, filename in files if filename in QLOG_FILENAMES)
+      except StopIteration:
+        qlog_path = None
+
       try:
         camera_path = next(path for path, filename in files if filename in CAMERA_FILENAMES)
       except StopIteration:
         camera_path = None
 
-      segments.append(RouteSegment(segment, log_path, camera_path))
+      segments.append(RouteSegment(segment, log_path, qlog_path, camera_path))
 
     if len(segments) == 0:
       raise ValueError('Could not find segments for route {} in data directory {}'.format(self.route_name, data_dir))
     return sorted(segments, key=lambda seg: seg.canonical_name.segment_num)
 
 class RouteSegment(object):
-  def __init__(self, name, log_path, camera_path):
+  def __init__(self, name, log_path, qlog_path, camera_path):
     self._name = RouteSegmentName(name)
     self.log_path = log_path
+    self.qlog_path = qlog_path
     self.camera_path = camera_path
 
   @property

tools/plotjuggler/juggle.py

@@ -10,7 +10,6 @@ from common.basedir import BASEDIR
 from selfdrive.test.process_replay.compare_logs import save_log
 from tools.lib.route import Route
 from tools.lib.logreader import LogReader
-from tools.lib.url_file import URLFile
 
 
 def load_segment(segment_name):
@@ -26,14 +25,26 @@ def juggle_file(fn):
   juggle_dir = os.path.dirname(os.path.realpath(__file__))
   subprocess.call(f"bin/plotjuggler -d {fn}", shell=True, env=env, cwd=juggle_dir)
 
-def juggle_route(route_name):
+def juggle_route(route_name, segment_number):
   r = Route(route_name)
-  all_data = []
 
-  pool = multiprocessing.Pool(24)
+  logs = r.log_paths()
+  if segment_number is not None:
+    logs = logs[segment_number:segment_number+1]
+
+  if None in logs:
+    fallback_answer = input("At least one of the rlogs in this segment does not exist, would you like to use the qlogs? (y/n) : ")
+    if fallback_answer == 'y':
+      logs = r.qlog_paths()
+      if segment_number is not None:
+        logs = logs[segment_number:segment_number+1]
+    else:
+      print(f"Please try a different {'segment' if segment_number is not None else 'route'}")
+      return
 
   all_data = []
-  for d in pool.map(load_segment, r.log_paths()):
+  pool = multiprocessing.Pool(24)
+  for d in pool.map(load_segment, logs):
     all_data += d
 
   tempfile = NamedTemporaryFile(suffix='.rlog')
@@ -42,17 +53,6 @@ def juggle_route(route_name):
 
   juggle_file(tempfile.name)
 
-def juggle_segment(route_name, segment_nr):
-  r = Route(route_name)
-  lp = r.log_paths()[segment_nr]
-
-  if lp is None:
-    print("This segment does not exist, please try a different one")
-    return
-
-  uf = URLFile(lp)
-  juggle_file(uf.name)
-
 def get_arg_parser():
   parser = argparse.ArgumentParser(description="PlotJuggler plugin for reading rlogs",
                                    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
@@ -63,8 +63,9 @@ def get_arg_parser():
 
 if __name__ == "__main__":
 
-  args = get_arg_parser().parse_args(sys.argv[1:])    
+  arg_parser = get_arg_parser()
-  if args.segment_number is None:
+  if len(sys.argv) == 1:
-    juggle_route(args.route_name)
+    arg_parser.print_help()
-  else:
+    sys.exit()
-    juggle_segment(args.route_name, args.segment_number)
+  args = arg_parser.parse_args(sys.argv[1:])
+  juggle_route(args.route_name, args.segment_number)