diff --git a/tools/car_porting/auto_fingerprint.py b/tools/car_porting/auto_fingerprint.py index 1870dcba80..ef3efc1cf3 100755 --- a/tools/car_porting/auto_fingerprint.py +++ b/tools/car_porting/auto_fingerprint.py @@ -5,10 +5,9 @@ from collections import defaultdict from typing import Optional from openpilot.selfdrive.debug.format_fingerprints import format_brand_fw_versions -from openpilot.tools.lib.logreader import MultiLogIterator -from openpilot.tools.lib.route import Route from openpilot.selfdrive.car.fw_versions import match_fw_to_car from openpilot.selfdrive.car.interfaces import get_interface_attr +from openpilot.tools.lib.srreader import SegmentRangeReader, ReadMode ALL_FW_VERSIONS = get_interface_attr("FW_VERSIONS") @@ -25,8 +24,7 @@ if __name__ == "__main__": parser.add_argument("platform", help="The platform, or leave empty to auto-determine using fuzzy", default=None, nargs='?') args = parser.parse_args() - route = Route(args.route) - lr = MultiLogIterator(route.qlog_paths()) + lr = SegmentRangeReader(args.route, ReadMode.QLOG) carFw = None carVin = None diff --git a/tools/car_porting/examples/subaru_steer_temp_fault.ipynb b/tools/car_porting/examples/subaru_steer_temp_fault.ipynb index a20d7d5062..b60915e6ec 100644 --- a/tools/car_porting/examples/subaru_steer_temp_fault.ipynb +++ b/tools/car_porting/examples/subaru_steer_temp_fault.ipynb @@ -9,9 +9,7 @@ "# An example of searching through a database of segments for a specific condition, and plotting the results.\n", "\n", "segments = [\n", - " \"c3d1ccb52f5f9d65|2023-07-22--01-23-20--6\",\n", - " \"c3d1ccb52f5f9d65|2023-07-22--01-23-20--7\",\n", - " \"c3d1ccb52f5f9d65|2023-07-22--01-23-20--8\",\n", + " \"c3d1ccb52f5f9d65|2023-07-22--01-23-20/6:10\",\n", "]\n", "platform = \"SUBARU OUTBACK 6TH GEN\"" ] @@ -25,13 +23,12 @@ "import copy\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", - "from openpilot.tools.lib.logreader import logreader_from_route_or_segment\n", - "\n", - "\n", - "from selfdrive.car.subaru.values import CanBus, DBC\n", "\n", "from opendbc.can.parser import CANParser\n", "\n", + "from openpilot.selfdrive.car.subaru.values import CanBus, DBC\n", + "from openpilot.tools.lib.srreader import SegmentRangeReader\n", + "\n", "\"\"\"\n", "In this example, we search for positive transitions of Steer_Warning, which indicate that the EPS\n", "has stopped responding to our messages. This analysis would allow you to find the cause of these\n", @@ -39,8 +36,7 @@ "\"\"\"\n", "\n", "for segment in segments:\n", - " print(segment)\n", - " lr = logreader_from_route_or_segment(segment)\n", + " lr = SegmentRangeReader(segment)\n", "\n", " can_msgs = [msg for msg in lr if msg.which() == \"can\"]\n", "\n", diff --git a/tools/lib/helpers.py b/tools/lib/helpers.py index 067b64b6ac..539d1ce34f 100644 --- a/tools/lib/helpers.py +++ b/tools/lib/helpers.py @@ -7,8 +7,11 @@ TIME_FMT = "%Y-%m-%d--%H-%M-%S" class RE: DONGLE_ID = r'(?P[a-z0-9]{16})' TIMESTAMP = r'(?P[0-9]{4}-[0-9]{2}-[0-9]{2}--[0-9]{2}-[0-9]{2}-[0-9]{2})' - ROUTE_NAME = r'{}[|_/]{}'.format(DONGLE_ID, TIMESTAMP) + ROUTE_NAME = r'(?P{}[|_/]{})'.format(DONGLE_ID, TIMESTAMP) SEGMENT_NAME = r'{}(?:--|/)(?P[0-9]+)'.format(ROUTE_NAME) + INDEX = r'-?[0-9]+' + SLICE = r'(?P{})?:?(?P{})?:?(?P{})?'.format(INDEX, INDEX, INDEX) + SEGMENT_RANGE = r'{}(?:--|/)?(?P({}))?'.format(ROUTE_NAME, SLICE) BOOTLOG_NAME = ROUTE_NAME EXPLORER_FILE = r'^(?P{})--(?P[a-z]+\.[a-z0-9]+)$'.format(SEGMENT_NAME) diff --git a/tools/lib/route.py b/tools/lib/route.py index e37b7d4434..17048773b2 100644 --- a/tools/lib/route.py +++ b/tools/lib/route.py @@ -229,3 +229,25 @@ class SegmentName: def data_dir(self) -> Optional[str]: return self._data_dir def __str__(self) -> str: return self._canonical_name + + +class SegmentRange: + def __init__(self, segment_range: str): + self.m = re.fullmatch(RE.SEGMENT_RANGE, segment_range) + assert self.m, f"Segment range is not valid {segment_range}" + + @property + def route_name(self): + return self.m.group("route_name") + + @property + def dongle_id(self): + return self.m.group("dongle_id") + + @property + def timestamp(self): + return self.m.group("timestamp") + + @property + def _slice(self): + return self.m.group("slice") diff --git a/tools/lib/srreader.py b/tools/lib/srreader.py new file mode 100644 index 0000000000..5ad13352d9 --- /dev/null +++ b/tools/lib/srreader.py @@ -0,0 +1,81 @@ +import enum +import re +import numpy as np +from openpilot.selfdrive.test.openpilotci import get_url +from openpilot.tools.lib.helpers import RE +from openpilot.tools.lib.logreader import LogReader +from openpilot.tools.lib.route import Route, SegmentRange + +class ReadMode(enum.Enum): + RLOG = 0 # only read rlogs + QLOG = 1 # only read qlogs + #AUTO = 2 # default to rlogs, fallback to qlogs, not supported yet + + +def create_slice_from_string(s: str): + m = re.fullmatch(RE.SLICE, s) + assert m is not None, f"Invalid slice: {s}" + start, end, step = m.groups() + start = int(start) if start is not None else None + end = int(end) if end is not None else None + step = int(step) if step is not None else None + + if start is not None and ":" not in s and end is None and step is None: + return start + return slice(start, end, step) + + +def parse_slice(sr: SegmentRange): + route = Route(sr.route_name) + segs = np.arange(route.max_seg_number+1) + s = create_slice_from_string(sr._slice) + return segs[s] if isinstance(s, slice) else [segs[s]] + +def comma_api_source(sr: SegmentRange, mode=ReadMode.RLOG): + segs = parse_slice(sr) + route = Route(sr.route_name) + + log_paths = route.log_paths() if mode == ReadMode.RLOG else route.qlog_paths() + + for seg in segs: + yield LogReader(log_paths[seg]) + +def internal_source(sr: SegmentRange, mode=ReadMode.RLOG): + segs = parse_slice(sr) + + for seg in segs: + yield LogReader(f"cd:/{sr.dongle_id}/{sr.timestamp}/{seg}/{'rlog' if mode == ReadMode.RLOG else 'qlog'}.bz2") + +def openpilotci_source(sr: SegmentRange, mode=ReadMode.RLOG): + segs = parse_slice(sr) + + for seg in segs: + yield LogReader(get_url(sr.route_name, seg, 'rlog' if mode == ReadMode.RLOG else 'qlog')) + +def auto_source(sr: SegmentRange, mode=ReadMode.RLOG): + # Automatically determine viable source + + try: + next(internal_source(sr, mode)) + return internal_source(sr, mode) + except Exception: + pass + + try: + next(openpilotci_source(sr, mode)) + return openpilotci_source(sr, mode) + except Exception: + pass + + return comma_api_source(sr, mode) + + +class SegmentRangeReader: + def __init__(self, segment_range: str, mode=ReadMode.RLOG, source=auto_source): + sr = SegmentRange(segment_range) + self.lrs = source(sr, mode) + + def __iter__(self): + for lr in self.lrs: + for m in lr: + yield m diff --git a/tools/lib/tests/test_srreader.py b/tools/lib/tests/test_srreader.py new file mode 100644 index 0000000000..2a017be6ae --- /dev/null +++ b/tools/lib/tests/test_srreader.py @@ -0,0 +1,63 @@ +import numpy as np +import unittest +from parameterized import parameterized + +from openpilot.tools.lib.route import SegmentRange +from openpilot.tools.lib.srreader import ReadMode, SegmentRangeReader, parse_slice + +NUM_SEGS = 17 # number of segments in the test route +ALL_SEGS = list(np.arange(NUM_SEGS)) +TEST_ROUTE = "344c5c15b34f2d8a/2024-01-03--09-37-12" + +class TestSegmentRangeReader(unittest.TestCase): + @parameterized.expand([ + (f"{TEST_ROUTE}", ALL_SEGS), + (f"{TEST_ROUTE.replace('/', '|')}", ALL_SEGS), + (f"{TEST_ROUTE}--0", [0]), + (f"{TEST_ROUTE}--5", [5]), + (f"{TEST_ROUTE}/0", [0]), + (f"{TEST_ROUTE}/5", [5]), + (f"{TEST_ROUTE}/0:10", ALL_SEGS[0:10]), + (f"{TEST_ROUTE}/0:0", []), + (f"{TEST_ROUTE}/4:6", ALL_SEGS[4:6]), + (f"{TEST_ROUTE}/0:-1", ALL_SEGS[0:-1]), + (f"{TEST_ROUTE}/:5", ALL_SEGS[:5]), + (f"{TEST_ROUTE}/2:", ALL_SEGS[2:]), + (f"{TEST_ROUTE}/2:-1", ALL_SEGS[2:-1]), + (f"{TEST_ROUTE}/-1", [ALL_SEGS[-1]]), + (f"{TEST_ROUTE}/-2", [ALL_SEGS[-2]]), + (f"{TEST_ROUTE}/-2:-1", ALL_SEGS[-2:-1]), + (f"{TEST_ROUTE}/-4:-2", ALL_SEGS[-4:-2]), + (f"{TEST_ROUTE}/:10:2", ALL_SEGS[:10:2]), + (f"{TEST_ROUTE}/5::2", ALL_SEGS[5::2]), + ]) + def test_parse_slice(self, segment_range, expected): + sr = SegmentRange(segment_range) + segs = parse_slice(sr) + self.assertListEqual(list(segs), expected) + + @parameterized.expand([ + (f"{TEST_ROUTE}//",), + (f"{TEST_ROUTE}---",), + (f"{TEST_ROUTE}/-4:--2",), + (f"{TEST_ROUTE}/-a",), + (f"{TEST_ROUTE}/0:1:2:3",), + (f"{TEST_ROUTE}/:::3",), + ]) + def test_bad_ranges(self, segment_range): + with self.assertRaises(AssertionError): + sr = SegmentRange(segment_range) + parse_slice(sr) + + @parameterized.expand([ + (ReadMode.QLOG, 11643), + (ReadMode.RLOG, 70577), + ]) + def test_modes(self, mode, expected): + lr = SegmentRangeReader(TEST_ROUTE+"/0", mode) + + self.assertEqual(len(list(lr)), expected) + + +if __name__ == "__main__": + unittest.main()