From 54da96dbdf23e4bf19ddf96b178418cbb068d584 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Sch=C3=A4fer?= Date: Thu, 17 Jul 2025 18:22:36 -0700 Subject: [PATCH 01/19] Route: add name format (#35743) * needed for now * TODO * Add TODO --- tools/lib/route.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/lib/route.py b/tools/lib/route.py index 0a4700f083..98a943e5e5 100644 --- a/tools/lib/route.py +++ b/tools/lib/route.py @@ -241,6 +241,10 @@ class SegmentName: @property def canonical_name(self) -> str: return self._canonical_name + #TODO should only use one name + @property + def data_name(self) -> str: return f"{self._route_name.canonical_name}/{self._num}" + @property def dongle_id(self) -> str: return self._route_name.dongle_id From c553c1f87205b475214952256e3a19d63b273bf4 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 17 Jul 2025 19:54:38 -0700 Subject: [PATCH 02/19] LogReader: support passing list of sources (#35749) * far too long * this is a better experience * no rename for now --- selfdrive/car/tests/test_models.py | 9 ++++----- tools/lib/logreader.py | 17 +++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/selfdrive/car/tests/test_models.py b/selfdrive/car/tests/test_models.py index b918f4c29f..cf7dd3146f 100644 --- a/selfdrive/car/tests/test_models.py +++ b/selfdrive/car/tests/test_models.py @@ -4,7 +4,6 @@ import pytest import random import unittest # noqa: TID251 from collections import defaultdict, Counter -from functools import partial import hypothesis.strategies as st from hypothesis import Phase, given, settings from parameterized import parameterized_class @@ -23,7 +22,7 @@ from openpilot.selfdrive.pandad import can_capnp_to_list from openpilot.selfdrive.test.helpers import read_segment_list from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT from openpilot.tools.lib.logreader import LogReader, LogsUnavailable, openpilotci_source_zst, openpilotci_source, internal_source, \ - internal_source_zst, comma_api_source, auto_source + internal_source_zst, comma_api_source from openpilot.tools.lib.route import SegmentName SafetyModel = car.CarParams.SafetyModel @@ -125,9 +124,9 @@ class TestCarModelBase(unittest.TestCase): segment_range = f"{cls.test_route.route}/{seg}" try: - source = partial(auto_source, sources=[internal_source, internal_source_zst] if len(INTERNAL_SEG_LIST) else \ - [openpilotci_source_zst, openpilotci_source, comma_api_source]) - lr = LogReader(segment_range, source=source, sort_by_time=True) + sources = ([internal_source, internal_source_zst] if len(INTERNAL_SEG_LIST) else + [openpilotci_source_zst, openpilotci_source, comma_api_source]) + lr = LogReader(segment_range, sources=sources, sort_by_time=True) return cls.get_testing_data_from_logreader(lr) except (LogsUnavailable, AssertionError): pass diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index 34d3e5ea9f..b0266baa2d 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -186,7 +186,7 @@ def openpilotci_source_zst(sr: SegmentRange, mode: ReadMode) -> list[LogPath]: return openpilotci_source(sr, mode, "zst") -def comma_car_segments_source(sr: SegmentRange, mode=ReadMode.RLOG) -> list[LogPath]: +def comma_car_segments_source(sr: SegmentRange, mode: ReadMode = ReadMode.RLOG) -> list[LogPath]: return [get_comma_segments_url(sr.route_name, seg) for seg in sr.seg_idxs] @@ -213,13 +213,10 @@ def check_source(source: Source, *args) -> list[LogPath]: return files -def auto_source(sr: SegmentRange, mode=ReadMode.RLOG, sources: list[Source] = None) -> list[LogPath]: +def auto_source(sr: SegmentRange, sources: list[Source], mode: ReadMode = ReadMode.RLOG) -> list[LogPath]: if mode == ReadMode.SANITIZED: return comma_car_segments_source(sr, mode) - if sources is None: - sources = [internal_source, internal_source_zst, openpilotci_source, openpilotci_source_zst, - comma_api_source, comma_car_segments_source, testing_closet_source] exceptions = {} # for automatic fallback modes, auto_source needs to first check if rlogs exist for any source @@ -267,7 +264,7 @@ class LogReader: sr = SegmentRange(identifier) mode = self.default_mode if sr.selector is None else ReadMode(sr.selector) - identifiers = self.source(sr, mode) + identifiers = auto_source(sr, self.sources, mode) invalid_count = len(list(get_invalid_files(identifiers))) assert invalid_count == 0, (f"{invalid_count}/{len(identifiers)} invalid log(s) found, please ensure all logs " + @@ -275,9 +272,13 @@ class LogReader: return identifiers def __init__(self, identifier: str | list[str], default_mode: ReadMode = ReadMode.RLOG, - source: Source = auto_source, sort_by_time=False, only_union_types=False): + sources: list[Source] = None, sort_by_time=False, only_union_types=False): + if sources is None: + sources = [internal_source, internal_source_zst, openpilotci_source, openpilotci_source_zst, + comma_api_source, comma_car_segments_source, testing_closet_source] + self.default_mode = default_mode - self.source = source + self.sources = sources self.identifier = identifier if isinstance(identifier, str): self.identifier = [identifier] From 5339a13c34680997983456b1055c072c76211f7c Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 17 Jul 2025 21:42:47 -0700 Subject: [PATCH 03/19] clean up --- tools/lib/filereader.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/lib/filereader.py b/tools/lib/filereader.py index 0bb4abd2fa..d342ad20da 100644 --- a/tools/lib/filereader.py +++ b/tools/lib/filereader.py @@ -1,13 +1,17 @@ import os +import functools import posixpath import socket from urllib.parse import urlparse +from tenacity import retry, retry_if_exception_type, stop_after_attempt from openpilot.tools.lib.url_file import URLFile DATA_ENDPOINT = os.getenv("DATA_ENDPOINT", "http://data-raw.comma.internal/") +# @functools.cache +# @retry(retry=retry_if_exception_type(TimeoutError), stop=stop_after_attempt(3), reraise=True) def internal_source_available(url=DATA_ENDPOINT): if os.path.isdir(url): return True From 239646b20b8c445f9f561b582e0132c97c66116f Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 17 Jul 2025 21:43:21 -0700 Subject: [PATCH 04/19] Revert "clean up" This reverts commit 5339a13c34680997983456b1055c072c76211f7c. don't push to master, kids --- tools/lib/filereader.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/lib/filereader.py b/tools/lib/filereader.py index d342ad20da..0bb4abd2fa 100644 --- a/tools/lib/filereader.py +++ b/tools/lib/filereader.py @@ -1,17 +1,13 @@ import os -import functools import posixpath import socket from urllib.parse import urlparse -from tenacity import retry, retry_if_exception_type, stop_after_attempt from openpilot.tools.lib.url_file import URLFile DATA_ENDPOINT = os.getenv("DATA_ENDPOINT", "http://data-raw.comma.internal/") -# @functools.cache -# @retry(retry=retry_if_exception_type(TimeoutError), stop=stop_after_attempt(3), reraise=True) def internal_source_available(url=DATA_ENDPOINT): if os.path.isdir(url): return True From cbba571845f921fd79eed43ac97e71867ce24492 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Sch=C3=A4fer?= Date: Thu, 17 Jul 2025 22:00:12 -0700 Subject: [PATCH 05/19] LeTR14h (#35746) f497b976-1ad4-4ef9-a6b8-172c68df12ea/400 --- selfdrive/modeld/models/driving_policy.onnx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfdrive/modeld/models/driving_policy.onnx b/selfdrive/modeld/models/driving_policy.onnx index 2582a9f1f5..ee9a9b6ec1 100644 --- a/selfdrive/modeld/models/driving_policy.onnx +++ b/selfdrive/modeld/models/driving_policy.onnx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cddefa5dbe21c60858f0d321d12d1ed4b126d05a7563f651bdf021674be63d64 +oid sha256:ef059460a95076f9a8600abb8c9d56c8c3b7384505b158064e2e875458b965bb size 15701037 From 862a8162150e0722078fe8c6b12f0272c21f280f Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 17 Jul 2025 23:12:08 -0700 Subject: [PATCH 06/19] LogReader: add data endpoint option (#35751) * add data endpoint option * clean up --- tools/lib/filereader.py | 4 ++-- tools/lib/logreader.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tools/lib/filereader.py b/tools/lib/filereader.py index 0bb4abd2fa..46fad7db7a 100644 --- a/tools/lib/filereader.py +++ b/tools/lib/filereader.py @@ -8,14 +8,14 @@ from openpilot.tools.lib.url_file import URLFile DATA_ENDPOINT = os.getenv("DATA_ENDPOINT", "http://data-raw.comma.internal/") -def internal_source_available(url=DATA_ENDPOINT): +def internal_source_available(url: str) -> bool: if os.path.isdir(url): return True try: hostname = urlparse(url).hostname port = urlparse(url).port or 80 - with socket.socket(socket.AF_INET,socket.SOCK_STREAM) as s: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.settimeout(0.5) s.connect((hostname, port)) return True diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index b0266baa2d..3b8370c59a 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -19,7 +19,7 @@ from cereal import log as capnp_log from openpilot.common.swaglog import cloudlog from openpilot.tools.lib.comma_car_segments import get_url as get_comma_segments_url from openpilot.tools.lib.openpilotci import get_url -from openpilot.tools.lib.filereader import FileReader, file_exists, internal_source_available +from openpilot.tools.lib.filereader import DATA_ENDPOINT, FileReader, file_exists, internal_source_available from openpilot.tools.lib.route import Route, SegmentRange from openpilot.tools.lib.log_time_series import msgs_to_time_series @@ -157,12 +157,13 @@ def comma_api_source(sr: SegmentRange, mode: ReadMode) -> list[LogPath]: return apply_strategy(mode, rlog_paths, qlog_paths, valid_file=valid_file) -def internal_source(sr: SegmentRange, mode: ReadMode, file_ext: str = "bz2") -> list[LogPath]: - if not internal_source_available(): +def internal_source(sr: SegmentRange, mode: ReadMode, file_ext: str = "bz2", + endpoint_url: str = DATA_ENDPOINT) -> list[LogPath]: + if not internal_source_available(endpoint_url): raise InternalUnavailableException def get_internal_url(sr: SegmentRange, seg, file): - return f"cd:/{sr.dongle_id}/{sr.log_id}/{seg}/{file}.{file_ext}" + return f"{endpoint_url.rstrip('/')}/{sr.dongle_id}/{sr.log_id}/{seg}/{file}.{file_ext}" # TODO: list instead of using static URLs to support routes with multiple file extensions rlog_paths = [get_internal_url(sr, seg, "rlog") for seg in sr.seg_idxs] @@ -171,8 +172,9 @@ def internal_source(sr: SegmentRange, mode: ReadMode, file_ext: str = "bz2") -> return apply_strategy(mode, rlog_paths, qlog_paths) -def internal_source_zst(sr: SegmentRange, mode: ReadMode, file_ext: str = "zst") -> list[LogPath]: - return internal_source(sr, mode, file_ext) +def internal_source_zst(sr: SegmentRange, mode: ReadMode, file_ext: str = "zst", + endpoint_url: str = DATA_ENDPOINT) -> list[LogPath]: + return internal_source(sr, mode, file_ext, endpoint_url) def openpilotci_source(sr: SegmentRange, mode: ReadMode, file_ext: str = "bz2") -> list[LogPath]: From b50b351b15012001d2e63b3c953952be997ae010 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 18 Jul 2025 02:17:11 -0700 Subject: [PATCH 07/19] LogReader: remove commaCarSegments selector (#35750) * a source should not be a readmode... and readmode should be logtype * fix that * fixup --- tools/car_porting/examples/find_segments_with_message.ipynb | 6 +++--- tools/car_porting/examples/ford_vin_fingerprint.ipynb | 6 +++--- tools/car_porting/examples/hkg_canfd_gear_message.ipynb | 6 +++--- tools/lib/comma_car_segments.py | 3 ++- tools/lib/helpers.py | 2 +- tools/lib/logreader.py | 4 ---- 6 files changed, 12 insertions(+), 15 deletions(-) diff --git a/tools/car_porting/examples/find_segments_with_message.ipynb b/tools/car_porting/examples/find_segments_with_message.ipynb index 4e688cc65b..af17bde52b 100644 --- a/tools/car_porting/examples/find_segments_with_message.ipynb +++ b/tools/car_porting/examples/find_segments_with_message.ipynb @@ -76,7 +76,7 @@ " if platform not in database:\n", " print(f\"No segments available for {platform}\")\n", " continue\n", - " \n", + "\n", " all_segments = database[platform]\n", " NUM_SEGMENTS = min(len(all_segments), MAX_SEGS_PER_PLATFORM)\n", " TEST_SEGMENTS.extend(random.sample(all_segments, NUM_SEGMENTS))\n", @@ -147,7 +147,7 @@ } ], "source": [ - "from openpilot.tools.lib.logreader import LogReader\n", + "from openpilot.tools.lib.logreader import LogReader, comma_car_segments_source\n", "from tqdm.notebook import tqdm, tnrange\n", "\n", "# Example search for CAN ignition messages\n", @@ -169,7 +169,7 @@ "progress_bar = tnrange(len(TEST_SEGMENTS), desc=\"segments searched\")\n", "\n", "for segment in TEST_SEGMENTS:\n", - " lr = LogReader(segment)\n", + " lr = LogReader(segment, sources=[comma_car_segments_source])\n", " CP = lr.first(\"carParams\")\n", " if CP is None:\n", " progress_bar.update()\n", diff --git a/tools/car_porting/examples/ford_vin_fingerprint.ipynb b/tools/car_porting/examples/ford_vin_fingerprint.ipynb index 7b0dd656da..6b806d22d2 100644 --- a/tools/car_porting/examples/ford_vin_fingerprint.ipynb +++ b/tools/car_porting/examples/ford_vin_fingerprint.ipynb @@ -20,7 +20,7 @@ "source": [ "\"\"\"In this example, we use the public comma car segments database to check if vin fingerprinting is feasible for ford.\"\"\"\n", "\n", - "from openpilot.tools.lib.logreader import LogReader\n", + "from openpilot.tools.lib.logreader import LogReader, comma_car_segments_source\n", "from openpilot.tools.lib.comma_car_segments import get_comma_car_segments_database\n", "from opendbc.car.ford.values import CAR\n", "\n", @@ -100,7 +100,7 @@ " if platform not in database:\n", " print(f\"Skipping platform: {platform}, no data available\")\n", " continue\n", - " \n", + "\n", " all_segments = database[platform]\n", "\n", " NUM_SEGMENTS = min(len(all_segments), MAX_SEGS_PER_PLATFORM)\n", @@ -110,7 +110,7 @@ " segments = random.sample(all_segments, NUM_SEGMENTS)\n", "\n", " for segment in segments:\n", - " lr = LogReader(segment)\n", + " lr = LogReader(segment, sources=[comma_car_segments_source])\n", " CP = lr.first(\"carParams\")\n", " if \"FORD\" not in CP.carFingerprint:\n", " print(segment, CP.carFingerprint)\n", diff --git a/tools/car_porting/examples/hkg_canfd_gear_message.ipynb b/tools/car_porting/examples/hkg_canfd_gear_message.ipynb index 5fdbdda684..f0bca8decc 100644 --- a/tools/car_porting/examples/hkg_canfd_gear_message.ipynb +++ b/tools/car_porting/examples/hkg_canfd_gear_message.ipynb @@ -72,7 +72,7 @@ " #if platform not in database:\n", " # print(f\"Skipping platform: {platform}, no data available\")\n", " # continue\n", - " \n", + "\n", " all_segments = database[platform]\n", "\n", " NUM_SEGMENTS = min(len(all_segments), MAX_SEGS_PER_PLATFORM)\n", @@ -198,12 +198,12 @@ "from opendbc.car.hyundai.hyundaicanfd import CanBus\n", "\n", "from openpilot.selfdrive.pandad import can_capnp_to_list\n", - "from openpilot.tools.lib.logreader import LogReader\n", + "from openpilot.tools.lib.logreader import LogReader, comma_car_segments_source\n", "\n", "message_names = [\"GEAR_SHIFTER\", \"ACCELERATOR\", \"GEAR\", \"GEAR_ALT\", \"GEAR_ALT_2\"]\n", "\n", "for segment in TEST_SEGMENTS:\n", - " lr = LogReader(segment)\n", + " lr = LogReader(segment, sources=[comma_car_segments_source])\n", " CP = lr.first(\"carParams\")\n", " if CP is None:\n", " continue\n", diff --git a/tools/lib/comma_car_segments.py b/tools/lib/comma_car_segments.py index 88496ae4d5..cd19356d66 100644 --- a/tools/lib/comma_car_segments.py +++ b/tools/lib/comma_car_segments.py @@ -14,7 +14,8 @@ def get_comma_car_segments_database(): ret = {} for platform in database: - ret[MIGRATION.get(platform, platform)] = database[platform] + # TODO: remove this when commaCarSegments is updated to remove selector + ret[MIGRATION.get(platform, platform)] = [s.rstrip('/s') for s in database[platform]] return ret diff --git a/tools/lib/helpers.py b/tools/lib/helpers.py index 7c34e17cb1..8c976e7ecc 100644 --- a/tools/lib/helpers.py +++ b/tools/lib/helpers.py @@ -9,7 +9,7 @@ class RE: INDEX = r'-?[0-9]+' SLICE = fr'(?P{INDEX})?:?(?P{INDEX})?:?(?P{INDEX})?' - SEGMENT_RANGE = fr'{ROUTE_NAME}(?:(--|/)(?P({SLICE})))?(?:/(?P([qras])))?' + SEGMENT_RANGE = fr'{ROUTE_NAME}(?:(--|/)(?P({SLICE})))?(?:/(?P([qra])))?' BOOTLOG_NAME = ROUTE_NAME diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index 3b8370c59a..2045f0cbb5 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -96,7 +96,6 @@ class _LogFileReader: class ReadMode(enum.StrEnum): RLOG = "r" # only read rlogs QLOG = "q" # only read qlogs - SANITIZED = "s" # read from the commaCarSegments database AUTO = "a" # default to rlogs, fallback to qlogs AUTO_INTERACTIVE = "i" # default to rlogs, fallback to qlogs with a prompt from the user @@ -216,9 +215,6 @@ def check_source(source: Source, *args) -> list[LogPath]: def auto_source(sr: SegmentRange, sources: list[Source], mode: ReadMode = ReadMode.RLOG) -> list[LogPath]: - if mode == ReadMode.SANITIZED: - return comma_car_segments_source(sr, mode) - exceptions = {} # for automatic fallback modes, auto_source needs to first check if rlogs exist for any source From 618a25a612f7ae745f1b5b37a31aae9cc1e6daae Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 18 Jul 2025 05:15:55 -0700 Subject: [PATCH 08/19] LogReader: simplify sourcing logic and improve AUTO performance (#35753) * sourcing supports any file type! * stuff * no camera for now * i guess these are fine without * rename * get rid of these too! * fix * fix * this is better * start to clean up! * better * holy holy * clean up sources * more robust * working but needs some clean up * clean up * remove some trash * nl * auto_source can only return if it finds acceptable logs from sources * double negative is confusing * default * list isn't hashable * fix typing * clean up * speed up -- test_models got zst before bz2 in openpilotci, so do that (some segments have both bz2 and zst!) * don't be a hero * same behavior for now --- selfdrive/car/tests/test_models.py | 6 +- tools/lib/filereader.py | 2 + tools/lib/logreader.py | 189 +++++++++++++---------------- 3 files changed, 89 insertions(+), 108 deletions(-) diff --git a/selfdrive/car/tests/test_models.py b/selfdrive/car/tests/test_models.py index cf7dd3146f..7966524458 100644 --- a/selfdrive/car/tests/test_models.py +++ b/selfdrive/car/tests/test_models.py @@ -21,8 +21,7 @@ from openpilot.common.basedir import BASEDIR from openpilot.selfdrive.pandad import can_capnp_to_list from openpilot.selfdrive.test.helpers import read_segment_list from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT -from openpilot.tools.lib.logreader import LogReader, LogsUnavailable, openpilotci_source_zst, openpilotci_source, internal_source, \ - internal_source_zst, comma_api_source +from openpilot.tools.lib.logreader import LogReader, LogsUnavailable, openpilotci_source, internal_source, comma_api_source from openpilot.tools.lib.route import SegmentName SafetyModel = car.CarParams.SafetyModel @@ -124,8 +123,7 @@ class TestCarModelBase(unittest.TestCase): segment_range = f"{cls.test_route.route}/{seg}" try: - sources = ([internal_source, internal_source_zst] if len(INTERNAL_SEG_LIST) else - [openpilotci_source_zst, openpilotci_source, comma_api_source]) + sources = [internal_source] if len(INTERNAL_SEG_LIST) else [openpilotci_source, comma_api_source] lr = LogReader(segment_range, sources=sources, sort_by_time=True) return cls.get_testing_data_from_logreader(lr) except (LogsUnavailable, AssertionError): diff --git a/tools/lib/filereader.py b/tools/lib/filereader.py index 46fad7db7a..70a5d3fe8c 100644 --- a/tools/lib/filereader.py +++ b/tools/lib/filereader.py @@ -1,6 +1,7 @@ import os import posixpath import socket +from functools import cache from urllib.parse import urlparse from openpilot.tools.lib.url_file import URLFile @@ -30,6 +31,7 @@ def resolve_name(fn): return fn +@cache def file_exists(fn): fn = resolve_name(fn) if fn.startswith(("http://", "https://")): diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index 2045f0cbb5..6d235b7c6b 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 import bz2 -from functools import cache, partial +from functools import partial import multiprocessing import capnp import enum @@ -13,6 +13,7 @@ import warnings import zstandard as zstd from collections.abc import Callable, Iterable, Iterator +from typing import cast from urllib.parse import parse_qs, urlparse from cereal import log as capnp_log @@ -100,9 +101,13 @@ class ReadMode(enum.StrEnum): AUTO_INTERACTIVE = "i" # default to rlogs, fallback to qlogs with a prompt from the user +class FileName(enum.Enum): + RLOG = ("rlog.zst", "rlog.bz2") + QLOG = ("qlog.zst", "qlog.bz2") + + LogPath = str | None -ValidFileCallable = Callable[[LogPath], bool] -Source = Callable[[SegmentRange, ReadMode], list[LogPath]] +Source = Callable[[SegmentRange, FileName], list[LogPath]] InternalUnavailableException = Exception("Internal source not available") @@ -111,129 +116,112 @@ class LogsUnavailable(Exception): pass -@cache -def default_valid_file(fn: LogPath) -> bool: - return fn is not None and file_exists(fn) - - -def auto_strategy(rlog_paths: list[LogPath], qlog_paths: list[LogPath], interactive: bool, valid_file: ValidFileCallable) -> list[LogPath]: - # auto select logs based on availability - missing_rlogs = [rlog is None or not valid_file(rlog) for rlog in rlog_paths].count(True) - if missing_rlogs != 0: - if interactive: - if input(f"{missing_rlogs}/{len(rlog_paths)} rlogs were not found, would you like to fallback to qlogs for those segments? (y/n) ").lower() != "y": - return rlog_paths - else: - cloudlog.warning(f"{missing_rlogs}/{len(rlog_paths)} rlogs were not found, falling back to qlogs for those segments...") - - return [rlog if valid_file(rlog) else (qlog if valid_file(qlog) else None) - for (rlog, qlog) in zip(rlog_paths, qlog_paths, strict=True)] - return rlog_paths - - -def apply_strategy(mode: ReadMode, rlog_paths: list[LogPath], qlog_paths: list[LogPath], valid_file: ValidFileCallable = default_valid_file) -> list[LogPath]: - if mode == ReadMode.RLOG: - return rlog_paths - elif mode == ReadMode.QLOG: - return qlog_paths - elif mode == ReadMode.AUTO: - return auto_strategy(rlog_paths, qlog_paths, False, valid_file) - elif mode == ReadMode.AUTO_INTERACTIVE: - return auto_strategy(rlog_paths, qlog_paths, True, valid_file) - raise ValueError(f"invalid mode: {mode}") - - -def comma_api_source(sr: SegmentRange, mode: ReadMode) -> list[LogPath]: +def comma_api_source(sr: SegmentRange, fns: FileName) -> list[LogPath]: route = Route(sr.route_name) - rlog_paths = [route.log_paths()[seg] for seg in sr.seg_idxs] - qlog_paths = [route.qlog_paths()[seg] for seg in sr.seg_idxs] - # comma api will have already checked if the file exists - def valid_file(fn): - return fn is not None - - return apply_strategy(mode, rlog_paths, qlog_paths, valid_file=valid_file) + if fns == FileName.RLOG: + return [route.log_paths()[seg] for seg in sr.seg_idxs] + else: + return [route.qlog_paths()[seg] for seg in sr.seg_idxs] -def internal_source(sr: SegmentRange, mode: ReadMode, file_ext: str = "bz2", - endpoint_url: str = DATA_ENDPOINT) -> list[LogPath]: +def internal_source(sr: SegmentRange, fns: FileName, endpoint_url: str = DATA_ENDPOINT) -> list[LogPath]: if not internal_source_available(endpoint_url): raise InternalUnavailableException def get_internal_url(sr: SegmentRange, seg, file): - return f"{endpoint_url.rstrip('/')}/{sr.dongle_id}/{sr.log_id}/{seg}/{file}.{file_ext}" - - # TODO: list instead of using static URLs to support routes with multiple file extensions - rlog_paths = [get_internal_url(sr, seg, "rlog") for seg in sr.seg_idxs] - qlog_paths = [get_internal_url(sr, seg, "qlog") for seg in sr.seg_idxs] - - return apply_strategy(mode, rlog_paths, qlog_paths) - + return f"{endpoint_url.rstrip('/')}/{sr.dongle_id}/{sr.log_id}/{seg}/{file}" -def internal_source_zst(sr: SegmentRange, mode: ReadMode, file_ext: str = "zst", - endpoint_url: str = DATA_ENDPOINT) -> list[LogPath]: - return internal_source(sr, mode, file_ext, endpoint_url) + return eval_source([[get_internal_url(sr, seg, fn) for fn in fns.value] for seg in sr.seg_idxs]) -def openpilotci_source(sr: SegmentRange, mode: ReadMode, file_ext: str = "bz2") -> list[LogPath]: - rlog_paths = [get_url(sr.route_name, seg, f"rlog.{file_ext}") for seg in sr.seg_idxs] - qlog_paths = [get_url(sr.route_name, seg, f"qlog.{file_ext}") for seg in sr.seg_idxs] +def openpilotci_source(sr: SegmentRange, fns: FileName) -> list[LogPath]: + return eval_source([[get_url(sr.route_name, seg, fn) for fn in fns.value] for seg in sr.seg_idxs]) - return apply_strategy(mode, rlog_paths, qlog_paths) +def comma_car_segments_source(sr: SegmentRange, fns: FileName) -> list[LogPath]: + return eval_source([get_comma_segments_url(sr.route_name, seg) for seg in sr.seg_idxs]) -def openpilotci_source_zst(sr: SegmentRange, mode: ReadMode) -> list[LogPath]: - return openpilotci_source(sr, mode, "zst") - -def comma_car_segments_source(sr: SegmentRange, mode: ReadMode = ReadMode.RLOG) -> list[LogPath]: - return [get_comma_segments_url(sr.route_name, seg) for seg in sr.seg_idxs] - - -def testing_closet_source(sr: SegmentRange, mode=ReadMode.RLOG) -> list[LogPath]: +def testing_closet_source(sr: SegmentRange, fns: FileName) -> list[LogPath]: if not internal_source_available('http://testing.comma.life'): raise InternalUnavailableException - return [f"http://testing.comma.life/download/{sr.route_name.replace('|', '/')}/{seg}/rlog" for seg in sr.seg_idxs] + return eval_source([f"http://testing.comma.life/download/{sr.route_name.replace('|', '/')}/{seg}/rlog" for seg in sr.seg_idxs]) -def direct_source(file_or_url: str) -> list[LogPath]: +def direct_source(file_or_url: str) -> list[str]: return [file_or_url] -def get_invalid_files(files): - for f in files: - if f is None or not file_exists(f): - yield f +def eval_source(files: list[list[str] | str]) -> list[LogPath]: + # Returns valid file URLs given a list of possible file URLs for each segment (e.g. rlog.bz2, rlog.zst) + valid_files: list[LogPath] = [] + + for urls in files: + if isinstance(urls, str): + urls = [urls] + for url in urls: + if file_exists(url): + valid_files.append(url) + break + else: + valid_files.append(None) -def check_source(source: Source, *args) -> list[LogPath]: - files = source(*args) - assert len(files) > 0, "No files on source" - assert next(get_invalid_files(files), False) is False, "Some files are invalid" - return files + return valid_files -def auto_source(sr: SegmentRange, sources: list[Source], mode: ReadMode = ReadMode.RLOG) -> list[LogPath]: +def auto_source(identifier: str, sources: list[Source], default_mode: ReadMode) -> list[str]: exceptions = {} - # for automatic fallback modes, auto_source needs to first check if rlogs exist for any source - if mode in [ReadMode.AUTO, ReadMode.AUTO_INTERACTIVE]: + sr = SegmentRange(identifier) + mode = default_mode if sr.selector is None else ReadMode(sr.selector) + + if mode == ReadMode.QLOG: + try_fns = [FileName.QLOG] + else: + try_fns = [FileName.RLOG] + + # If selector allows it, fallback to qlogs + if mode in (ReadMode.AUTO, ReadMode.AUTO_INTERACTIVE): + try_fns.append(FileName.QLOG) + + # Build a dict of valid files as we evaluate each source. May contain mix of rlogs, qlogs, and None. + # This function only returns when we've sourced all files, or throws an exception + valid_files: dict[int, LogPath] = {} + for fn in try_fns: for source in sources: try: - return check_source(source, sr, ReadMode.RLOG) - except Exception: - pass + files = source(sr, fn) - # Automatically determine viable source - for source in sources: - try: - return check_source(source, sr, mode) - except Exception as e: - exceptions[source.__name__] = e + # Check every source returns an expected number of files + assert len(files) == len(valid_files) or len(valid_files) == 0, f"Source {source.__name__} returned unexpected number of files" + + # Build a dict of valid files + for idx, f in enumerate(files): + if valid_files.get(idx) is None: + valid_files[idx] = f + + # We've found all files, return them + if all(f is not None for f in valid_files.values()): + return cast(list[str], list(valid_files.values())) + + except Exception as e: + exceptions[source.__name__] = e - raise LogsUnavailable("auto_source could not find any valid source, exceptions for sources:\n - " + - "\n - ".join([f"{k}: {repr(v)}" for k, v in exceptions.items()])) + if fn == try_fns[0]: + missing_logs = list(valid_files.values()).count(None) + if mode == ReadMode.AUTO: + cloudlog.warning(f"{missing_logs}/{len(valid_files)} rlogs were not found, falling back to qlogs for those segments...") + elif mode == ReadMode.AUTO_INTERACTIVE: + if input(f"{missing_logs}/{len(valid_files)} rlogs were not found, would you like to fallback to qlogs for those segments? (y/N) ").lower() != "y": + break + + missing_logs = list(valid_files.values()).count(None) + raise LogsUnavailable(f"{missing_logs}/{len(valid_files)} logs were not found, please ensure all logs " + + "are uploaded. You can fall back to qlogs with '/a' selector at the end of the route name.\n\n" + + "Exceptions for sources:\n - " + "\n - ".join([f"{k}: {repr(v)}" for k, v in exceptions.items()])) def parse_indirect(identifier: str) -> str: @@ -250,7 +238,7 @@ def parse_direct(identifier: str): class LogReader: - def _parse_identifier(self, identifier: str) -> list[LogPath]: + def _parse_identifier(self, identifier: str) -> list[str]: # useradmin, etc. identifier = parse_indirect(identifier) @@ -259,21 +247,14 @@ class LogReader: if direct_parsed is not None: return direct_source(identifier) - sr = SegmentRange(identifier) - mode = self.default_mode if sr.selector is None else ReadMode(sr.selector) - - identifiers = auto_source(sr, self.sources, mode) - - invalid_count = len(list(get_invalid_files(identifiers))) - assert invalid_count == 0, (f"{invalid_count}/{len(identifiers)} invalid log(s) found, please ensure all logs " + - "are uploaded or auto fallback to qlogs with '/a' selector at the end of the route name.") + identifiers = auto_source(identifier, self.sources, self.default_mode) return identifiers def __init__(self, identifier: str | list[str], default_mode: ReadMode = ReadMode.RLOG, sources: list[Source] = None, sort_by_time=False, only_union_types=False): if sources is None: - sources = [internal_source, internal_source_zst, openpilotci_source, openpilotci_source_zst, - comma_api_source, comma_car_segments_source, testing_closet_source] + sources = [internal_source, openpilotci_source, comma_api_source, + comma_car_segments_source, testing_closet_source] self.default_mode = default_mode self.sources = sources From 22e54af238440a56a7f6b49b34e87250d958bc1e Mon Sep 17 00:00:00 2001 From: Dean Lee Date: Sat, 19 Jul 2025 00:21:46 +0800 Subject: [PATCH 09/19] cabana: fix bit mask calculation bug (#35755) Fix bit mask calculation bug --- tools/cabana/dbc/dbc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cabana/dbc/dbc.cc b/tools/cabana/dbc/dbc.cc index e9c869fce7..9b0de92218 100644 --- a/tools/cabana/dbc/dbc.cc +++ b/tools/cabana/dbc/dbc.cc @@ -109,7 +109,7 @@ void cabana::Msg::update() { mask[i] |= ((1ULL << sz) - 1) << shift; - bits -= size; + bits -= sz; i = sig->is_little_endian ? i - 1 : i + 1; } } From dbe172f7fd5dc371ef1f8bb8b1082ee70808902a Mon Sep 17 00:00:00 2001 From: Bruce Wayne Date: Fri, 18 Jul 2025 15:39:00 -0700 Subject: [PATCH 10/19] Fragile xx code needs this --- tools/lib/route.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/lib/route.py b/tools/lib/route.py index 98a943e5e5..c8e2895fde 100644 --- a/tools/lib/route.py +++ b/tools/lib/route.py @@ -10,9 +10,9 @@ from openpilot.tools.lib.auth_config import get_token from openpilot.tools.lib.api import APIError, CommaApi from openpilot.tools.lib.helpers import RE -QLOG_FILENAMES = ['qlog', 'qlog.bz2', 'qlog.zst'] -QCAMERA_FILENAMES = ['qcamera.ts'] -LOG_FILENAMES = ['rlog', 'rlog.bz2', 'raw_log.bz2', 'rlog.zst'] +QLOG_FILENAMES = ['qlog.bz2', 'qlog.zst', 'qlog'] +QCAMERA_FILENAMES = ['qcamera.ts', 'qcamera.ts.gz'] +LOG_FILENAMES = ['rlog.bz2', 'raw_log.bz2', 'rlog.zst', 'rlog'] CAMERA_FILENAMES = ['fcamera.hevc', 'video.hevc'] DCAMERA_FILENAMES = ['dcamera.hevc'] ECAMERA_FILENAMES = ['ecamera.hevc'] From 4a5385a5b09c6dc837e6511ed67935191896ac15 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 18 Jul 2025 16:03:30 -0700 Subject: [PATCH 11/19] LogReader: support connect URLs (#35757) * support connect * clean up --- tools/lib/logreader.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index 6d235b7c6b..0642e50eba 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -227,7 +227,24 @@ def auto_source(identifier: str, sources: list[Source], default_mode: ReadMode) def parse_indirect(identifier: str) -> str: if "useradmin.comma.ai" in identifier: query = parse_qs(urlparse(identifier).query) - return query["onebox"][0] + identifier = query["onebox"][0] + elif "connect.comma.ai" in identifier: + path = urlparse(identifier).path.strip("/").split("/") + path = ['/'.join(path[:2]), *path[2:]] # recombine log id + + identifier = path[0] + if len(path) > 2: + # convert url with seconds to segments + start, end = int(path[1]) // 60, int(path[2]) // 60 + 1 + identifier = f"{identifier}/{start}:{end}" + + # add selector if it exists + if len(path) > 3: + identifier += f"/{path[3]}" + else: + # add selector if it exists + identifier = "/".join(path) + return identifier From 43ad1754da7c0136788afa70f038d2f1b4dffd58 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 18 Jul 2025 16:17:01 -0700 Subject: [PATCH 12/19] LogReader: fix broken test (#35758) fix lr test --- tools/lib/tests/test_logreader.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/lib/tests/test_logreader.py b/tools/lib/tests/test_logreader.py index 230b6a65ea..11bdd33ccf 100644 --- a/tools/lib/tests/test_logreader.py +++ b/tools/lib/tests/test_logreader.py @@ -10,7 +10,7 @@ import requests from parameterized import parameterized from cereal import log as capnp_log -from openpilot.tools.lib.logreader import LogIterable, LogReader, comma_api_source, parse_indirect, ReadMode, InternalUnavailableException +from openpilot.tools.lib.logreader import LogsUnavailable, LogIterable, LogReader, comma_api_source, parse_indirect, ReadMode, InternalUnavailableException from openpilot.tools.lib.route import SegmentRange from openpilot.tools.lib.url_file import URLFileException @@ -193,17 +193,17 @@ class TestLogReader: with subtests.test("interactive_yes"): mocker.patch("sys.stdin", new=io.StringIO("y\n")) - lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, source=comma_api_source) + lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, sources=[comma_api_source]) log_len = len(list(lr)) assert qlog_len == log_len with subtests.test("interactive_no"): mocker.patch("sys.stdin", new=io.StringIO("n\n")) - with pytest.raises(AssertionError): - lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, source=comma_api_source) + with pytest.raises(LogsUnavailable): + lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, sources=[comma_api_source]) with subtests.test("non_interactive"): - lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO, source=comma_api_source) + lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO, sources=[comma_api_source]) log_len = len(list(lr)) assert qlog_len == log_len From 278781e5af2bbd33caf4c06eaff9384a2d7aab0f Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 18 Jul 2025 17:00:39 -0700 Subject: [PATCH 13/19] LogReader: extend FileName (#35760) extend filename --- tools/lib/logreader.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index 0642e50eba..61b4c5a502 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -104,6 +104,10 @@ class ReadMode(enum.StrEnum): class FileName(enum.Enum): RLOG = ("rlog.zst", "rlog.bz2") QLOG = ("qlog.zst", "qlog.bz2") + QCAMERA = ("qcamera.ts",) + FCAMERA = ("fcamera.hevc",) + ECAMERA = ("ecamera.hevc",) + DCAMERA = ("dcamera.hevc",) LogPath = str | None From fbbad834f3d085f2aa13e8a98f0cc0f75d7d27c6 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 18 Jul 2025 18:18:02 -0700 Subject: [PATCH 14/19] LogReader: cache internal source available (#35761) * cache * fix * match behavioir --- tools/lib/filereader.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/lib/filereader.py b/tools/lib/filereader.py index 70a5d3fe8c..02f5fd1b95 100644 --- a/tools/lib/filereader.py +++ b/tools/lib/filereader.py @@ -2,6 +2,7 @@ import os import posixpath import socket from functools import cache +from openpilot.common.retry import retry from urllib.parse import urlparse from openpilot.tools.lib.url_file import URLFile @@ -9,6 +10,8 @@ from openpilot.tools.lib.url_file import URLFile DATA_ENDPOINT = os.getenv("DATA_ENDPOINT", "http://data-raw.comma.internal/") +@cache +@retry(delay=0.0) def internal_source_available(url: str) -> bool: if os.path.isdir(url): return True From 8ead5db67081a4d75fe2d1c6d123adecceea71fe Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 18 Jul 2025 19:19:26 -0700 Subject: [PATCH 15/19] raylib: add style guide (#35764) * add style guide * more --- system/ui/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/system/ui/README.md b/system/ui/README.md index 85d32bfd6c..b124ae4d85 100644 --- a/system/ui/README.md +++ b/system/ui/README.md @@ -8,3 +8,8 @@ Quick start: * set `SCALE=1.5` to scale the entire UI by 1.5x * https://www.raylib.com/cheatsheet/cheatsheet.html * https://electronstudio.github.io/raylib-python-cffi/README.html#quickstart + +Style guide: +* All graphical elements should subclass [`Widget`](/system/ui/widgets/__init__.py). + * Prefer a stateful widget over a function for easy migration from QT +* All internal class variables and functions should be prefixed with `_` From 48892e339d17f290433eb8c749ab48e30b753e18 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 18 Jul 2025 19:21:04 -0700 Subject: [PATCH 16/19] LogReader: remove testing closet source (#35763) rm --- tools/lib/logreader.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index 61b4c5a502..b4d5a4324b 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -148,12 +148,6 @@ def comma_car_segments_source(sr: SegmentRange, fns: FileName) -> list[LogPath]: return eval_source([get_comma_segments_url(sr.route_name, seg) for seg in sr.seg_idxs]) -def testing_closet_source(sr: SegmentRange, fns: FileName) -> list[LogPath]: - if not internal_source_available('http://testing.comma.life'): - raise InternalUnavailableException - return eval_source([f"http://testing.comma.life/download/{sr.route_name.replace('|', '/')}/{seg}/rlog" for seg in sr.seg_idxs]) - - def direct_source(file_or_url: str) -> list[str]: return [file_or_url] @@ -274,8 +268,7 @@ class LogReader: def __init__(self, identifier: str | list[str], default_mode: ReadMode = ReadMode.RLOG, sources: list[Source] = None, sort_by_time=False, only_union_types=False): if sources is None: - sources = [internal_source, openpilotci_source, comma_api_source, - comma_car_segments_source, testing_closet_source] + sources = [internal_source, openpilotci_source, comma_api_source, comma_car_segments_source] self.default_mode = default_mode self.sources = sources From 1936c42ee71de44d72022f03f7f73f54c08d3c1c Mon Sep 17 00:00:00 2001 From: Maxime Desroches Date: Fri, 18 Jul 2025 20:09:35 -0700 Subject: [PATCH 17/19] Port `reset` to new raylib events handling (#35762) * test * more * type * type * order * _ * __ --- system/ui/reset.py | 58 ++++++++++++++++++++----------------- system/ui/widgets/button.py | 49 +++++++++++++++++++++++++++++-- 2 files changed, 78 insertions(+), 29 deletions(-) diff --git a/system/ui/reset.py b/system/ui/reset.py index aedca004b9..2178741ab5 100755 --- a/system/ui/reset.py +++ b/system/ui/reset.py @@ -1,14 +1,15 @@ #!/usr/bin/env python3 import os -import pyray as rl import sys import threading from enum import IntEnum +import pyray as rl + from openpilot.system.hardware import PC from openpilot.system.ui.lib.application import gui_app, FontWeight from openpilot.system.ui.widgets import Widget -from openpilot.system.ui.widgets.button import gui_button, ButtonStyle +from openpilot.system.ui.widgets.button import Button, ButtonStyle from openpilot.system.ui.widgets.label import gui_label, gui_text_box NVME = "/dev/nvme0n1" @@ -31,8 +32,15 @@ class ResetState(IntEnum): class Reset(Widget): def __init__(self, mode): super().__init__() - self.mode = mode - self.reset_state = ResetState.NONE + self._mode = mode + self._reset_state = ResetState.NONE + self._cancel_button = Button("Cancel", self._cancel_callback) + self._confirm_button = Button("Confirm", self._confirm, button_style=ButtonStyle.PRIMARY) + self._reboot_button = Button("Reboot", lambda: os.system("sudo reboot")) + self._render_status = True + + def _cancel_callback(self): + self._render_status = False def _do_erase(self): if PC: @@ -50,10 +58,10 @@ class Reset(Widget): if rm == 0 or fmt == 0: os.system("sudo reboot") else: - self.reset_state = ResetState.FAILED + self._reset_state = ResetState.FAILED def start_reset(self): - self.reset_state = ResetState.RESETTING + self._reset_state = ResetState.RESETTING threading.Timer(0.1, self._do_erase).start() def _render(self, rect: rl.Rectangle): @@ -61,42 +69,38 @@ class Reset(Widget): gui_label(label_rect, "System Reset", 100, font_weight=FontWeight.BOLD) text_rect = rl.Rectangle(rect.x + 140, rect.y + 140, rect.width - 280, rect.height - 90 - 100) - gui_text_box(text_rect, self.get_body_text(), 90) + gui_text_box(text_rect, self._get_body_text(), 90) button_height = 160 button_spacing = 50 button_top = rect.y + rect.height - button_height button_width = (rect.width - button_spacing) / 2.0 - if self.reset_state != ResetState.RESETTING: - if self.mode == ResetMode.RECOVER or self.reset_state == ResetState.FAILED: - if gui_button(rl.Rectangle(rect.x, button_top, button_width, button_height), "Reboot"): - os.system("sudo reboot") - elif self.mode == ResetMode.USER_RESET: - if gui_button(rl.Rectangle(rect.x, button_top, button_width, button_height), "Cancel"): - return False + if self._reset_state != ResetState.RESETTING: + if self._mode == ResetMode.RECOVER or self._reset_state == ResetState.FAILED: + self._reboot_button.render(rl.Rectangle(rect.x, button_top, rect.width, button_height)) + elif self._mode == ResetMode.USER_RESET: + self._cancel_button.render(rl.Rectangle(rect.x, button_top, button_width, button_height)) - if self.reset_state != ResetState.FAILED: - if gui_button(rl.Rectangle(rect.x + button_width + 50, button_top, button_width, button_height), - "Confirm", button_style=ButtonStyle.PRIMARY): - self.confirm() + if self._reset_state != ResetState.FAILED: + self._confirm_button.render(rl.Rectangle(rect.x + button_width + 50, button_top, button_width, button_height)) - return True + return self._render_status - def confirm(self): - if self.reset_state == ResetState.CONFIRM: + def _confirm(self): + if self._reset_state == ResetState.CONFIRM: self.start_reset() else: - self.reset_state = ResetState.CONFIRM + self._reset_state = ResetState.CONFIRM - def get_body_text(self): - if self.reset_state == ResetState.CONFIRM: + def _get_body_text(self): + if self._reset_state == ResetState.CONFIRM: return "Are you sure you want to reset your device?" - if self.reset_state == ResetState.RESETTING: + if self._reset_state == ResetState.RESETTING: return "Resetting device...\nThis may take up to a minute." - if self.reset_state == ResetState.FAILED: + if self._reset_state == ResetState.FAILED: return "Reset failed. Reboot to try again." - if self.mode == ResetMode.RECOVER: + if self._mode == ResetMode.RECOVER: return "Unable to mount data partition. Partition may be corrupted. Press confirm to erase and reset your device." return "System reset triggered. Press confirm to erase all content and settings. Press cancel to resume boot." diff --git a/system/ui/widgets/button.py b/system/ui/widgets/button.py index cfab842e93..2be56e6dd5 100644 --- a/system/ui/widgets/button.py +++ b/system/ui/widgets/button.py @@ -1,7 +1,11 @@ -import pyray as rl +from collections.abc import Callable from enum import IntEnum -from openpilot.system.ui.lib.application import gui_app, FontWeight + +import pyray as rl + +from openpilot.system.ui.lib.application import gui_app, FontWeight, MousePos from openpilot.system.ui.lib.text_measure import measure_text_cached +from openpilot.system.ui.widgets import Widget class ButtonStyle(IntEnum): @@ -148,3 +152,44 @@ def gui_button( rl.draw_text_ex(font, text, text_pos, font_size, 0, color) return result + + +class Button(Widget): + def __init__(self, + text: str, + click_callback: Callable[[], None] = None, + font_size: int = DEFAULT_BUTTON_FONT_SIZE, + font_weight: FontWeight = FontWeight.MEDIUM, + button_style: ButtonStyle = ButtonStyle.NORMAL, + border_radius: int = 10, + ): + + super().__init__() + self._text = text + self._click_callback = click_callback + self._label_font = gui_app.font(FontWeight.SEMI_BOLD) + self._button_style = button_style + self._font_size = font_size + self._border_radius = border_radius + self._font_size = font_size + self._text_color = BUTTON_TEXT_COLOR[button_style] + self._text_size = measure_text_cached(gui_app.font(font_weight), text, font_size) + + def _handle_mouse_release(self, mouse_pos: MousePos): + if self._click_callback: + print(f"Button clicked: {self._text}") + self._click_callback() + + def _get_background_color(self) -> rl.Color: + if self._is_pressed: + return BUTTON_PRESSED_BACKGROUND_COLORS[self._button_style] + else: + return BUTTON_BACKGROUND_COLORS[self._button_style] + + def _render(self, _): + roundness = self._border_radius / (min(self._rect.width, self._rect.height) / 2) + rl.draw_rectangle_rounded(self._rect, roundness, 10, self._get_background_color()) + + text_pos = rl.Vector2(0, self._rect.y + (self._rect.height - self._text_size.y) // 2) + text_pos.x = self._rect.x + (self._rect.width - self._text_size.x) // 2 + rl.draw_text_ex(self._label_font, self._text, text_pos, self._font_size, 0, self._text_color) From e1fa04b6783662aed56f2b516214af06293e438b Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 18 Jul 2025 20:20:34 -0700 Subject: [PATCH 18/19] Remove qcamera.ts.gz filename (#35766) rm --- tools/lib/route.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/lib/route.py b/tools/lib/route.py index c8e2895fde..5b5ccf3bc0 100644 --- a/tools/lib/route.py +++ b/tools/lib/route.py @@ -11,7 +11,7 @@ from openpilot.tools.lib.api import APIError, CommaApi from openpilot.tools.lib.helpers import RE QLOG_FILENAMES = ['qlog.bz2', 'qlog.zst', 'qlog'] -QCAMERA_FILENAMES = ['qcamera.ts', 'qcamera.ts.gz'] +QCAMERA_FILENAMES = ['qcamera.ts'] LOG_FILENAMES = ['rlog.bz2', 'raw_log.bz2', 'rlog.zst', 'rlog'] CAMERA_FILENAMES = ['fcamera.hevc', 'video.hevc'] DCAMERA_FILENAMES = ['dcamera.hevc'] From ffc8ea5a867ab74c58d64d06247c31ecb9c8b784 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Fri, 18 Jul 2025 20:29:22 -0700 Subject: [PATCH 19/19] LogReader: use file name constants (#35765) use common --- tools/lib/logreader.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index b4d5a4324b..473ba989b8 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -21,7 +21,7 @@ from openpilot.common.swaglog import cloudlog from openpilot.tools.lib.comma_car_segments import get_url as get_comma_segments_url from openpilot.tools.lib.openpilotci import get_url from openpilot.tools.lib.filereader import DATA_ENDPOINT, FileReader, file_exists, internal_source_available -from openpilot.tools.lib.route import Route, SegmentRange +from openpilot.tools.lib.route import QCAMERA_FILENAMES, CAMERA_FILENAMES, DCAMERA_FILENAMES, ECAMERA_FILENAMES, Route, SegmentRange from openpilot.tools.lib.log_time_series import msgs_to_time_series LogMessage = type[capnp._DynamicStructReader] @@ -104,10 +104,10 @@ class ReadMode(enum.StrEnum): class FileName(enum.Enum): RLOG = ("rlog.zst", "rlog.bz2") QLOG = ("qlog.zst", "qlog.bz2") - QCAMERA = ("qcamera.ts",) - FCAMERA = ("fcamera.hevc",) - ECAMERA = ("ecamera.hevc",) - DCAMERA = ("dcamera.hevc",) + QCAMERA = tuple(QCAMERA_FILENAMES) + FCAMERA = tuple(CAMERA_FILENAMES) + ECAMERA = tuple(ECAMERA_FILENAMES) + DCAMERA = tuple(DCAMERA_FILENAMES) LogPath = str | None