From b50b351b15012001d2e63b3c953952be997ae010 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 18 Jul 2025 02:17:11 -0700 Subject: [PATCH] LogReader: remove commaCarSegments selector (#35750) * a source should not be a readmode... and readmode should be logtype * fix that * fixup --- tools/car_porting/examples/find_segments_with_message.ipynb | 6 +++--- tools/car_porting/examples/ford_vin_fingerprint.ipynb | 6 +++--- tools/car_porting/examples/hkg_canfd_gear_message.ipynb | 6 +++--- tools/lib/comma_car_segments.py | 3 ++- tools/lib/helpers.py | 2 +- tools/lib/logreader.py | 4 ---- 6 files changed, 12 insertions(+), 15 deletions(-) diff --git a/tools/car_porting/examples/find_segments_with_message.ipynb b/tools/car_porting/examples/find_segments_with_message.ipynb index 4e688cc65b..af17bde52b 100644 --- a/tools/car_porting/examples/find_segments_with_message.ipynb +++ b/tools/car_porting/examples/find_segments_with_message.ipynb @@ -76,7 +76,7 @@ " if platform not in database:\n", " print(f\"No segments available for {platform}\")\n", " continue\n", - " \n", + "\n", " all_segments = database[platform]\n", " NUM_SEGMENTS = min(len(all_segments), MAX_SEGS_PER_PLATFORM)\n", " TEST_SEGMENTS.extend(random.sample(all_segments, NUM_SEGMENTS))\n", @@ -147,7 +147,7 @@ } ], "source": [ - "from openpilot.tools.lib.logreader import LogReader\n", + "from openpilot.tools.lib.logreader import LogReader, comma_car_segments_source\n", "from tqdm.notebook import tqdm, tnrange\n", "\n", "# Example search for CAN ignition messages\n", @@ -169,7 +169,7 @@ "progress_bar = tnrange(len(TEST_SEGMENTS), desc=\"segments searched\")\n", "\n", "for segment in TEST_SEGMENTS:\n", - " lr = LogReader(segment)\n", + " lr = LogReader(segment, sources=[comma_car_segments_source])\n", " CP = lr.first(\"carParams\")\n", " if CP is None:\n", " progress_bar.update()\n", diff --git a/tools/car_porting/examples/ford_vin_fingerprint.ipynb b/tools/car_porting/examples/ford_vin_fingerprint.ipynb index 7b0dd656da..6b806d22d2 100644 --- a/tools/car_porting/examples/ford_vin_fingerprint.ipynb +++ b/tools/car_porting/examples/ford_vin_fingerprint.ipynb @@ -20,7 +20,7 @@ "source": [ "\"\"\"In this example, we use the public comma car segments database to check if vin fingerprinting is feasible for ford.\"\"\"\n", "\n", - "from openpilot.tools.lib.logreader import LogReader\n", + "from openpilot.tools.lib.logreader import LogReader, comma_car_segments_source\n", "from openpilot.tools.lib.comma_car_segments import get_comma_car_segments_database\n", "from opendbc.car.ford.values import CAR\n", "\n", @@ -100,7 +100,7 @@ " if platform not in database:\n", " print(f\"Skipping platform: {platform}, no data available\")\n", " continue\n", - " \n", + "\n", " all_segments = database[platform]\n", "\n", " NUM_SEGMENTS = min(len(all_segments), MAX_SEGS_PER_PLATFORM)\n", @@ -110,7 +110,7 @@ " segments = random.sample(all_segments, NUM_SEGMENTS)\n", "\n", " for segment in segments:\n", - " lr = LogReader(segment)\n", + " lr = LogReader(segment, sources=[comma_car_segments_source])\n", " CP = lr.first(\"carParams\")\n", " if \"FORD\" not in CP.carFingerprint:\n", " print(segment, CP.carFingerprint)\n", diff --git a/tools/car_porting/examples/hkg_canfd_gear_message.ipynb b/tools/car_porting/examples/hkg_canfd_gear_message.ipynb index 5fdbdda684..f0bca8decc 100644 --- a/tools/car_porting/examples/hkg_canfd_gear_message.ipynb +++ b/tools/car_porting/examples/hkg_canfd_gear_message.ipynb @@ -72,7 +72,7 @@ " #if platform not in database:\n", " # print(f\"Skipping platform: {platform}, no data available\")\n", " # continue\n", - " \n", + "\n", " all_segments = database[platform]\n", "\n", " NUM_SEGMENTS = min(len(all_segments), MAX_SEGS_PER_PLATFORM)\n", @@ -198,12 +198,12 @@ "from opendbc.car.hyundai.hyundaicanfd import CanBus\n", "\n", "from openpilot.selfdrive.pandad import can_capnp_to_list\n", - "from openpilot.tools.lib.logreader import LogReader\n", + "from openpilot.tools.lib.logreader import LogReader, comma_car_segments_source\n", "\n", "message_names = [\"GEAR_SHIFTER\", \"ACCELERATOR\", \"GEAR\", \"GEAR_ALT\", \"GEAR_ALT_2\"]\n", "\n", "for segment in TEST_SEGMENTS:\n", - " lr = LogReader(segment)\n", + " lr = LogReader(segment, sources=[comma_car_segments_source])\n", " CP = lr.first(\"carParams\")\n", " if CP is None:\n", " continue\n", diff --git a/tools/lib/comma_car_segments.py b/tools/lib/comma_car_segments.py index 88496ae4d5..cd19356d66 100644 --- a/tools/lib/comma_car_segments.py +++ b/tools/lib/comma_car_segments.py @@ -14,7 +14,8 @@ def get_comma_car_segments_database(): ret = {} for platform in database: - ret[MIGRATION.get(platform, platform)] = database[platform] + # TODO: remove this when commaCarSegments is updated to remove selector + ret[MIGRATION.get(platform, platform)] = [s.rstrip('/s') for s in database[platform]] return ret diff --git a/tools/lib/helpers.py b/tools/lib/helpers.py index 7c34e17cb1..8c976e7ecc 100644 --- a/tools/lib/helpers.py +++ b/tools/lib/helpers.py @@ -9,7 +9,7 @@ class RE: INDEX = r'-?[0-9]+' SLICE = fr'(?P{INDEX})?:?(?P{INDEX})?:?(?P{INDEX})?' - SEGMENT_RANGE = fr'{ROUTE_NAME}(?:(--|/)(?P({SLICE})))?(?:/(?P([qras])))?' + SEGMENT_RANGE = fr'{ROUTE_NAME}(?:(--|/)(?P({SLICE})))?(?:/(?P([qra])))?' BOOTLOG_NAME = ROUTE_NAME diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index 3b8370c59a..2045f0cbb5 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -96,7 +96,6 @@ class _LogFileReader: class ReadMode(enum.StrEnum): RLOG = "r" # only read rlogs QLOG = "q" # only read qlogs - SANITIZED = "s" # read from the commaCarSegments database AUTO = "a" # default to rlogs, fallback to qlogs AUTO_INTERACTIVE = "i" # default to rlogs, fallback to qlogs with a prompt from the user @@ -216,9 +215,6 @@ def check_source(source: Source, *args) -> list[LogPath]: def auto_source(sr: SegmentRange, sources: list[Source], mode: ReadMode = ReadMode.RLOG) -> list[LogPath]: - if mode == ReadMode.SANITIZED: - return comma_car_segments_source(sr, mode) - exceptions = {} # for automatic fallback modes, auto_source needs to first check if rlogs exist for any source