plotjuggler: support segment names (#23263)

* plotjuggler: support segment names

* update docs

* generic parser

* convert segment number to int when parsing

* add SegmentName

Co-authored-by: Willem Melching <willem.melching@gmail.com>
old-commit-hash: 71132edf17

tools/lib/route.py

@@ -7,9 +7,10 @@ from itertools import chain
 from tools.lib.auth_config import get_token
 from tools.lib.api import CommaApi
 
-SEGMENT_NAME_RE = r'[a-z0-9]{16}[|_][0-9]{4}-[0-9]{2}-[0-9]{2}--[0-9]{2}-[0-9]{2}-[0-9]{2}--[0-9]+'
-EXPLORER_FILE_RE = r'^({})--([a-z]+\.[a-z0-9]+)$'.format(SEGMENT_NAME_RE)
-OP_SEGMENT_DIR_RE = r'^({})$'.format(SEGMENT_NAME_RE)
+ROUTE_NAME_RE = r'(?P<dongle_id>[a-z0-9]{16})[|_/](?P<timestamp>[0-9]{4}-[0-9]{2}-[0-9]{2}--[0-9]{2}-[0-9]{2}-[0-9]{2})'
+SEGMENT_NAME_RE = r'{}(?:--|/)(?P<segment_num>[0-9]+)'.format(ROUTE_NAME_RE)
+EXPLORER_FILE_RE = r'^(?P<segment_name>{})--(?P<file_name>[a-z]+\.[a-z0-9]+)$'.format(SEGMENT_NAME_RE)
+OP_SEGMENT_DIR_RE = r'^(?P<segment_name>{})$'.format(SEGMENT_NAME_RE)
 
 QLOG_FILENAMES = ['qlog.bz2']
 QCAMERA_FILENAMES = ['qcamera.ts']
@@ -19,47 +20,51 @@ DCAMERA_FILENAMES = ['dcamera.hevc']
 ECAMERA_FILENAMES = ['ecamera.hevc']
 
 class Route:
-  def __init__(self, route_name, data_dir=None):
+  def __init__(self, name, data_dir=None):
+    self._name = RouteName(name)
     self.files = None
-    self.route_name = route_name.replace('_', '|')
     if data_dir is not None:
       self._segments = self._get_segments_local(data_dir)
     else:
       self._segments = self._get_segments_remote()
-    self.max_seg_number = self._segments[-1].canonical_name.segment_num
+    self.max_seg_number = self._segments[-1].name.segment_num
+
+  @property
+  def name(self):
+    return self._name
 
   @property
   def segments(self):
     return self._segments
 
   def log_paths(self):
-    log_path_by_seg_num = {s.canonical_name.segment_num: s.log_path for s in self._segments}
+    log_path_by_seg_num = {s.name.segment_num: s.log_path for s in self._segments}
     return [log_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
 
   def qlog_paths(self):
-    qlog_path_by_seg_num = {s.canonical_name.segment_num: s.qlog_path for s in self._segments}
+    qlog_path_by_seg_num = {s.name.segment_num: s.qlog_path for s in self._segments}
     return [qlog_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
 
   def camera_paths(self):
-    camera_path_by_seg_num = {s.canonical_name.segment_num: s.camera_path for s in self._segments}
+    camera_path_by_seg_num = {s.name.segment_num: s.camera_path for s in self._segments}
     return [camera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
 
   def dcamera_paths(self):
-    dcamera_path_by_seg_num = {s.canonical_name.segment_num: s.dcamera_path for s in self._segments}
+    dcamera_path_by_seg_num = {s.name.segment_num: s.dcamera_path for s in self._segments}
     return [dcamera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
 
   def ecamera_paths(self):
-    ecamera_path_by_seg_num = {s.canonical_name.segment_num: s.ecamera_path for s in self._segments}
+    ecamera_path_by_seg_num = {s.name.segment_num: s.ecamera_path for s in self._segments}
     return [ecamera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
 
   def qcamera_paths(self):
-    qcamera_path_by_seg_num = {s.canonical_name.segment_num: s.qcamera_path for s in self._segments}
+    qcamera_path_by_seg_num = {s.name.segment_num: s.qcamera_path for s in self._segments}
     return [qcamera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)]
 
   # TODO: refactor this, it's super repetitive
   def _get_segments_remote(self):
     api = CommaApi(get_token())
-    route_files = api.get('v1/route/' + self.route_name + '/files')
+    route_files = api.get('v1/route/' + self.name.canonical_name + '/files')
     self.files = list(chain.from_iterable(route_files.values()))
 
     segments = {}
@@ -67,7 +72,7 @@ class Route:
       _, dongle_id, time_str, segment_num, fn = urlparse(url).path.rsplit('/', maxsplit=4)
       segment_name = f'{dongle_id}|{time_str}--{segment_num}'
       if segments.get(segment_name):
-        segments[segment_name] = RouteSegment(
+        segments[segment_name] = Segment(
           segment_name,
           url if fn in LOG_FILENAMES else segments[segment_name].log_path,
           url if fn in QLOG_FILENAMES else segments[segment_name].qlog_path,
@@ -77,7 +82,7 @@ class Route:
           url if fn in QCAMERA_FILENAMES else segments[segment_name].qcamera_path,
         )
       else:
-        segments[segment_name] = RouteSegment(
+        segments[segment_name] = Segment(
           segment_name,
           url if fn in LOG_FILENAMES else None,
           url if fn in QLOG_FILENAMES else None,
@@ -87,7 +92,7 @@ class Route:
           url if fn in QCAMERA_FILENAMES else None,
         )
 
-    return sorted(segments.values(), key=lambda seg: seg.canonical_name.segment_num)
+    return sorted(segments.values(), key=lambda seg: seg.name.segment_num)
 
   def _get_segments_local(self, data_dir):
     files = os.listdir(data_dir)
@@ -99,20 +104,21 @@ class Route:
       op_match = re.match(OP_SEGMENT_DIR_RE, f)
 
       if explorer_match:
-        segment_name, fn = explorer_match.groups()
-        if segment_name.replace('_', '|').startswith(self.route_name):
+        segment_name = explorer_match.group('segment_name')
+        fn = explorer_match.group('file_name')
+        if segment_name.replace('_', '|').startswith(self.name.canonical_name):
           segment_files[segment_name].append((fullpath, fn))
       elif op_match and os.path.isdir(fullpath):
-        segment_name, = op_match.groups()
-        if segment_name.startswith(self.route_name):
+        segment_name = op_match.group('segment_name')
+        if segment_name.startswith(self.name.canonical_name):
           for seg_f in os.listdir(fullpath):
             segment_files[segment_name].append((os.path.join(fullpath, seg_f), seg_f))
-      elif f == self.route_name:
+      elif f == self.name.canonical_name:
         for seg_num in os.listdir(fullpath):
           if not seg_num.isdigit():
             continue
 
-          segment_name = f'{self.route_name}--{seg_num}'
+          segment_name = f'{self.name.canonical_name}--{seg_num}'
           for seg_f in os.listdir(os.path.join(fullpath, seg_num)):
             segment_files[segment_name].append((os.path.join(fullpath, seg_num, seg_f), seg_f))
 
@@ -149,15 +155,15 @@ class Route:
       except StopIteration:
         qcamera_path = None
 
-      segments.append(RouteSegment(segment, log_path, qlog_path, camera_path, dcamera_path, ecamera_path, qcamera_path))
+      segments.append(Segment(segment, log_path, qlog_path, camera_path, dcamera_path, ecamera_path, qcamera_path))
 
     if len(segments) == 0:
-      raise ValueError(f'Could not find segments for route {self.route_name} in data directory {data_dir}')
-    return sorted(segments, key=lambda seg: seg.canonical_name.segment_num)
+      raise ValueError(f'Could not find segments for route {self.name.canonical_name} in data directory {data_dir}')
+    return sorted(segments, key=lambda seg: seg.name.segment_num)
 
-class RouteSegment:
+class Segment:
   def __init__(self, name, log_path, qlog_path, camera_path, dcamera_path, ecamera_path, qcamera_path):
-    self._name = RouteSegmentName(name)
+    self._name = SegmentName(name)
     self.log_path = log_path
     self.qlog_path = qlog_path
     self.camera_path = camera_path
@@ -167,21 +173,55 @@ class RouteSegment:
 
   @property
   def name(self):
-    return str(self._name)
+    return self._name
+
+class RouteName:
+  def __init__(self, name_str: str):
+    self._name_str = name_str
+    delim = next(c for c in self._name_str if c in ("|", "/"))
+    self._dongle_id, self._time_str = self._name_str.split(delim)
+
+    assert len(self._dongle_id) == 16, self._name_str
+    assert len(self._time_str) == 20, self._name_str
+    self._canonical_name = f"{self._dongle_id}|{self._time_str}"
 
   @property
-  def canonical_name(self):
-    return self._name
+  def canonical_name(self) -> str: return self._canonical_name
+
+  @property
+  def dongle_id(self) -> str: return self._dongle_id
 
-class RouteSegmentName:
-  def __init__(self, name_str):
-    self._segment_name_str = name_str
-    self._route_name_str, num_str = self._segment_name_str.rsplit("--", 1)
-    self._num = int(num_str)
+  @property
+  def time_str(self) -> str: return self._time_str
+
+  def __str__(self) -> str: return self._canonical_name
+
+class SegmentName:
+  # TODO: add constructor that takes dongle_id, time_str, segment_num and then create instances
+  # of this class instead of manually constructing a segment name (use canonical_name prop instead)
+  def __init__(self, name_str: str, allow_route_name=False):
+    self._name_str = name_str
+    seg_num_delim = "--" if self._name_str.count("--") == 2 else "/"
+    name_parts = self._name_str.rsplit(seg_num_delim, 1)
+    if allow_route_name and len(name_parts) == 1:
+      name_parts.append("-1") # no segment number
+    self._route_name = RouteName(name_parts[0])
+    self._num = int(name_parts[1])
+    self._canonical_name = f"{self._route_name._dongle_id}|{self._route_name._time_str}--{self._num}"
+
+  @property
+  def canonical_name(self) -> str: return self._canonical_name
+
+  @property
+  def dongle_id(self) -> str: return self._route_name.dongle_id
+
+  @property
+  def time_str(self) -> str: return self._route_name.time_str
+
+  @property
+  def segment_num(self) -> int: return self._num
 
   @property
-  def segment_num(self):
-    return self._num
+  def route_name(self) -> RouteName: return self._route_name
 
-  def __str__(self):
-    return self._segment_name_str
+  def __str__(self) -> str: return self._canonical_name

tools/plotjuggler/README.md

@@ -13,14 +13,14 @@ Once you've cloned and are in openpilot, this command will download PlotJuggler
 ```
 $ ./juggle.py -h
 usage: juggle.py [-h] [--demo] [--qlog] [--can] [--stream] [--layout [LAYOUT]] [--install]
-                 [route_name] [segment_number] [segment_count]
+                 [route_or_segment_name] [segment_count]
 
 A helper to run PlotJuggler on openpilot routes
 
 positional arguments:
-  route_name         The route name to plot (cabana share URL accepted) (default: None)
-  segment_number     The index of the segment to plot (default: None)
-  segment_count      The number of segments to plot (default: 1)
+  route_or_segment_name
+                        The route or segment name to plot (cabana share URL accepted) (default: None)
+  segment_count         The number of segments to plot (default: None)
 
 optional arguments:
   -h, --help         show this help message and exit
@@ -32,10 +32,14 @@ optional arguments:
   --install          Install or update PlotJuggler + plugins (default: False)
 ```
 
-Example:
+Examples using route name:
 
 `./juggle.py "4cf7a6ad03080c90|2021-09-29--13-46-36"`
 
+Examples using segment name:
+
+`./juggle.py "4cf7a6ad03080c90|2021-09-29--13-46-36--1"`
+
 ## Streaming
 
 Explore live data from your car! Follow these steps to stream from your comma device to your laptop:

tools/plotjuggler/juggle.py

@@ -15,7 +15,7 @@ from selfdrive.test.process_replay.compare_logs import save_log
 from tools.lib.api import CommaApi
 from tools.lib.auth_config import get_token
 from tools.lib.robust_logreader import RobustLogReader
-from tools.lib.route import Route
+from tools.lib.route import Route, SegmentName
 from urllib.parse import urlparse, parse_qs
 
 juggle_dir = os.path.dirname(os.path.realpath(__file__))
@@ -74,29 +74,33 @@ def start_juggler(fn=None, dbc=None, layout=None):
   subprocess.call(cmd, shell=True, env=env, cwd=juggle_dir)
 
 
-def juggle_route(route_name, segment_number, segment_count, qlog, can, layout):
-  # TODO: abstract out the cabana stuff
-  if 'cabana' in route_name:
-    query = parse_qs(urlparse(route_name).query)
+def juggle_route(route_or_segment_name, segment_count, qlog, can, layout):
+  segment_start = 0
+  if 'cabana' in route_or_segment_name:
+    query = parse_qs(urlparse(route_or_segment_name).query)
     api = CommaApi(get_token())
     logs = api.get(f'v1/route/{query["route"][0]}/log_urls?sig={query["sig"][0]}&exp={query["exp"][0]}')
-  elif route_name.startswith("http://") or route_name.startswith("https://") or os.path.isfile(route_name):
-    logs = [route_name]
+  elif route_or_segment_name.startswith("http://") or route_or_segment_name.startswith("https://") or os.path.isfile(route_or_segment_name):
+    logs = [route_or_segment_name]
   else:
-    r = Route(route_name)
+    route_or_segment_name = SegmentName(route_or_segment_name, allow_route_name=True)
+    segment_start = max(route_or_segment_name.segment_num, 0)
+
+    if route_or_segment_name.segment_num != -1 and segment_count is None:
+      segment_count = 1
+
+    r = Route(route_or_segment_name.route_name.canonical_name)
     logs = r.qlog_paths() if qlog else r.log_paths()
 
-  if segment_number is not None:
-    logs = logs[segment_number:segment_number+segment_count]
+  segment_end = segment_start + segment_count if segment_count else -1
+  logs = logs[segment_start:segment_end]
 
   if None in logs:
     ans = input(f"{logs.count(None)}/{len(logs)} of the rlogs in this segment are missing, would you like to fall back to the qlogs? (y/n) ")
     if ans == 'y':
-      logs = r.qlog_paths()
-      if segment_number is not None:
-        logs = logs[segment_number:segment_number+segment_count]
+      logs = r.qlog_paths()[segment_start:segment_end]
     else:
-      print(f"Please try a different {'segment' if segment_number is not None else 'route'}")
+      print("Please try a different route or segment")
       return
 
   all_data = []
@@ -133,9 +137,9 @@ if __name__ == "__main__":
   parser.add_argument("--stream", action="store_true", help="Start PlotJuggler in streaming mode")
   parser.add_argument("--layout", nargs='?', help="Run PlotJuggler with a pre-defined layout")
   parser.add_argument("--install", action="store_true", help="Install or update PlotJuggler + plugins")
-  parser.add_argument("route_name", nargs='?', help="The route name to plot (cabana share URL accepted)")
-  parser.add_argument("segment_number", type=int, nargs='?', help="The index of the segment to plot")
-  parser.add_argument("segment_count", type=int, nargs='?', help="The number of segments to plot", default=1)
+  parser.add_argument("route_or_segment_name", nargs='?', help="The route or segment name to plot (cabana share URL accepted)")
+  parser.add_argument("segment_count", type=int, nargs='?', help="The number of segments to plot")
+
   if len(sys.argv) == 1:
     parser.print_help()
     sys.exit()
@@ -148,5 +152,5 @@ if __name__ == "__main__":
   if args.stream:
     start_juggler(layout=args.layout)
   else:
-    route = DEMO_ROUTE if args.demo else args.route_name.strip()
-    juggle_route(route, args.segment_number, args.segment_count, args.qlog, args.can, args.layout)
+    route_or_segment_name = DEMO_ROUTE if args.demo else args.route_or_segment_name.strip()
+    juggle_route(route_or_segment_name, args.segment_count, args.qlog, args.can, args.layout)