SegmentRangeReader: new format for reading multiple segments (#30940)

* segment range reader

* rename that

* revert that

* cleanup

* revert this for now

* revert this for now

* Fix + test

* rm that

* rm that

* use for auto_fingerprint

* simpler

* for notebook too

* match numpy indexing

* just use numpy directly

* remove that

* spacing

* spacing

* use qlog for auto fingerprint

* add 'read mode'

* pass in read mode

* add test for modes

* numpy indexing

* fix that case

* more examples

* fix the notebook

* cleanup the notebook

* cleaner

* fix those
old-commit-hash: 0d126e1e9e

tools/car_porting/auto_fingerprint.py

@@ -5,10 +5,9 @@ from collections import defaultdict
 from typing import Optional
 from openpilot.selfdrive.debug.format_fingerprints import format_brand_fw_versions
 
-from openpilot.tools.lib.logreader import MultiLogIterator
-from openpilot.tools.lib.route import Route
 from openpilot.selfdrive.car.fw_versions import match_fw_to_car
 from openpilot.selfdrive.car.interfaces import get_interface_attr
+from openpilot.tools.lib.srreader import SegmentRangeReader, ReadMode
 
 
 ALL_FW_VERSIONS = get_interface_attr("FW_VERSIONS")
@@ -25,8 +24,7 @@ if __name__ == "__main__":
   parser.add_argument("platform", help="The platform, or leave empty to auto-determine using fuzzy", default=None, nargs='?')
   args = parser.parse_args()
 
-  route = Route(args.route)
+  lr = SegmentRangeReader(args.route, ReadMode.QLOG)
-  lr = MultiLogIterator(route.qlog_paths())
 
   carFw = None
   carVin = None

tools/car_porting/examples/subaru_steer_temp_fault.ipynb

@@ -9,9 +9,7 @@
     "# An example of searching through a database of segments for a specific condition, and plotting the results.\n",
     "\n",
     "segments = [\n",
-    "  \"c3d1ccb52f5f9d65|2023-07-22--01-23-20--6\",\n",
+    "  \"c3d1ccb52f5f9d65|2023-07-22--01-23-20/6:10\",\n",
-    "  \"c3d1ccb52f5f9d65|2023-07-22--01-23-20--7\",\n",
-    "  \"c3d1ccb52f5f9d65|2023-07-22--01-23-20--8\",\n",
     "]\n",
     "platform = \"SUBARU OUTBACK 6TH GEN\""
    ]
@@ -25,13 +23,12 @@
     "import copy\n",
     "import matplotlib.pyplot as plt\n",
     "import numpy as np\n",
-    "from openpilot.tools.lib.logreader import logreader_from_route_or_segment\n",
-    "\n",
-    "\n",
-    "from selfdrive.car.subaru.values import CanBus, DBC\n",
     "\n",
     "from opendbc.can.parser import CANParser\n",
     "\n",
+    "from openpilot.selfdrive.car.subaru.values import CanBus, DBC\n",
+    "from openpilot.tools.lib.srreader import SegmentRangeReader\n",
+    "\n",
     "\"\"\"\n",
     "In this example, we search for positive transitions of Steer_Warning, which indicate that the EPS\n",
     "has stopped responding to our messages. This analysis would allow you to find the cause of these\n",
@@ -39,8 +36,7 @@
     "\"\"\"\n",
     "\n",
     "for segment in segments:\n",
-    "  print(segment)\n",
+    "  lr = SegmentRangeReader(segment)\n",
-    "  lr = logreader_from_route_or_segment(segment)\n",
     "\n",
     "  can_msgs = [msg for msg in lr if msg.which() == \"can\"]\n",
     "\n",

tools/lib/helpers.py

@@ -7,8 +7,11 @@ TIME_FMT = "%Y-%m-%d--%H-%M-%S"
 class RE:
   DONGLE_ID =  r'(?P<dongle_id>[a-z0-9]{16})'
   TIMESTAMP = r'(?P<timestamp>[0-9]{4}-[0-9]{2}-[0-9]{2}--[0-9]{2}-[0-9]{2}-[0-9]{2})'
-  ROUTE_NAME = r'{}[|_/]{}'.format(DONGLE_ID, TIMESTAMP)
+  ROUTE_NAME = r'(?P<route_name>{}[|_/]{})'.format(DONGLE_ID, TIMESTAMP)
   SEGMENT_NAME = r'{}(?:--|/)(?P<segment_num>[0-9]+)'.format(ROUTE_NAME)
+  INDEX = r'-?[0-9]+'
+  SLICE = r'(?P<start>{})?:?(?P<end>{})?:?(?P<step>{})?'.format(INDEX, INDEX, INDEX)
+  SEGMENT_RANGE = r'{}(?:--|/)?(?P<slice>({}))?'.format(ROUTE_NAME, SLICE)
   BOOTLOG_NAME = ROUTE_NAME
 
   EXPLORER_FILE = r'^(?P<segment_name>{})--(?P<file_name>[a-z]+\.[a-z0-9]+)$'.format(SEGMENT_NAME)

tools/lib/route.py

@@ -229,3 +229,25 @@ class SegmentName:
   def data_dir(self) -> Optional[str]: return self._data_dir
 
   def __str__(self) -> str: return self._canonical_name
+
+
+class SegmentRange:
+  def __init__(self, segment_range: str):
+    self.m = re.fullmatch(RE.SEGMENT_RANGE, segment_range)
+    assert self.m, f"Segment range is not valid {segment_range}"
+
+  @property
+  def route_name(self):
+    return self.m.group("route_name")
+
+  @property
+  def dongle_id(self):
+    return self.m.group("dongle_id")
+
+  @property
+  def timestamp(self):
+    return self.m.group("timestamp")
+
+  @property
+  def _slice(self):
+    return self.m.group("slice")

tools/lib/srreader.py

@@ -0,0 +1,81 @@
+import enum
+import re
+import numpy as np
+from openpilot.selfdrive.test.openpilotci import get_url
+from openpilot.tools.lib.helpers import RE
+from openpilot.tools.lib.logreader import LogReader
+from openpilot.tools.lib.route import Route, SegmentRange
+
+class ReadMode(enum.Enum):
+  RLOG = 0 # only read rlogs
+  QLOG = 1 # only read qlogs
+  #AUTO = 2 # default to rlogs, fallback to qlogs, not supported yet
+
+
+def create_slice_from_string(s: str):
+  m = re.fullmatch(RE.SLICE, s)
+  assert m is not None, f"Invalid slice: {s}"
+  start, end, step = m.groups()
+  start = int(start) if start is not None else None
+  end = int(end) if end is not None else None
+  step = int(step) if step is not None else None
+
+  if start is not None and ":" not in s and end is None and step is None:
+    return start
+  return slice(start, end, step)
+
+
+def parse_slice(sr: SegmentRange):
+  route = Route(sr.route_name)
+  segs = np.arange(route.max_seg_number+1)
+  s = create_slice_from_string(sr._slice)
+  return segs[s] if isinstance(s, slice) else [segs[s]]
+
+def comma_api_source(sr: SegmentRange, mode=ReadMode.RLOG):
+  segs = parse_slice(sr)
+  route = Route(sr.route_name)
+
+  log_paths = route.log_paths() if mode == ReadMode.RLOG else route.qlog_paths()
+
+  for seg in segs:
+    yield LogReader(log_paths[seg])
+
+def internal_source(sr: SegmentRange, mode=ReadMode.RLOG):
+  segs = parse_slice(sr)
+
+  for seg in segs:
+    yield LogReader(f"cd:/{sr.dongle_id}/{sr.timestamp}/{seg}/{'rlog' if mode == ReadMode.RLOG else 'qlog'}.bz2")
+
+def openpilotci_source(sr: SegmentRange, mode=ReadMode.RLOG):
+  segs = parse_slice(sr)
+
+  for seg in segs:
+    yield LogReader(get_url(sr.route_name, seg, 'rlog' if mode == ReadMode.RLOG else 'qlog'))
+
+def auto_source(sr: SegmentRange, mode=ReadMode.RLOG):
+  # Automatically determine viable source
+
+  try:
+    next(internal_source(sr, mode))
+    return internal_source(sr, mode)
+  except Exception:
+    pass
+
+  try:
+    next(openpilotci_source(sr, mode))
+    return openpilotci_source(sr, mode)
+  except Exception:
+    pass
+
+  return comma_api_source(sr, mode)
+
+
+class SegmentRangeReader:
+  def __init__(self, segment_range: str, mode=ReadMode.RLOG, source=auto_source):
+    sr = SegmentRange(segment_range)
+    self.lrs = source(sr, mode)
+
+  def __iter__(self):
+    for lr in self.lrs:
+      for m in lr:
+        yield m

tools/lib/tests/test_srreader.py

@@ -0,0 +1,63 @@
+import numpy as np
+import unittest
+from parameterized import parameterized
+
+from openpilot.tools.lib.route import SegmentRange
+from openpilot.tools.lib.srreader import ReadMode, SegmentRangeReader, parse_slice
+
+NUM_SEGS = 17 # number of segments in the test route
+ALL_SEGS = list(np.arange(NUM_SEGS))
+TEST_ROUTE = "344c5c15b34f2d8a/2024-01-03--09-37-12"
+
+class TestSegmentRangeReader(unittest.TestCase):
+  @parameterized.expand([
+    (f"{TEST_ROUTE}", ALL_SEGS),
+    (f"{TEST_ROUTE.replace('/', '|')}", ALL_SEGS),
+    (f"{TEST_ROUTE}--0", [0]),
+    (f"{TEST_ROUTE}--5", [5]),
+    (f"{TEST_ROUTE}/0", [0]),
+    (f"{TEST_ROUTE}/5", [5]),
+    (f"{TEST_ROUTE}/0:10", ALL_SEGS[0:10]),
+    (f"{TEST_ROUTE}/0:0", []),
+    (f"{TEST_ROUTE}/4:6", ALL_SEGS[4:6]),
+    (f"{TEST_ROUTE}/0:-1", ALL_SEGS[0:-1]),
+    (f"{TEST_ROUTE}/:5", ALL_SEGS[:5]),
+    (f"{TEST_ROUTE}/2:", ALL_SEGS[2:]),
+    (f"{TEST_ROUTE}/2:-1", ALL_SEGS[2:-1]),
+    (f"{TEST_ROUTE}/-1", [ALL_SEGS[-1]]),
+    (f"{TEST_ROUTE}/-2", [ALL_SEGS[-2]]),
+    (f"{TEST_ROUTE}/-2:-1", ALL_SEGS[-2:-1]),
+    (f"{TEST_ROUTE}/-4:-2", ALL_SEGS[-4:-2]),
+    (f"{TEST_ROUTE}/:10:2", ALL_SEGS[:10:2]),
+    (f"{TEST_ROUTE}/5::2", ALL_SEGS[5::2]),
+  ])
+  def test_parse_slice(self, segment_range, expected):
+    sr = SegmentRange(segment_range)
+    segs = parse_slice(sr)
+    self.assertListEqual(list(segs), expected)
+
+  @parameterized.expand([
+    (f"{TEST_ROUTE}//",),
+    (f"{TEST_ROUTE}---",),
+    (f"{TEST_ROUTE}/-4:--2",),
+    (f"{TEST_ROUTE}/-a",),
+    (f"{TEST_ROUTE}/0:1:2:3",),
+    (f"{TEST_ROUTE}/:::3",),
+  ])
+  def test_bad_ranges(self, segment_range):
+    with self.assertRaises(AssertionError):
+      sr = SegmentRange(segment_range)
+      parse_slice(sr)
+
+  @parameterized.expand([
+    (ReadMode.QLOG, 11643),
+    (ReadMode.RLOG, 70577),
+  ])
+  def test_modes(self, mode, expected):
+    lr = SegmentRangeReader(TEST_ROUTE+"/0", mode)
+
+    self.assertEqual(len(list(lr)), expected)
+
+
+if __name__ == "__main__":
+  unittest.main()