diff --git a/selfdrive/car/tests/test_models.py b/selfdrive/car/tests/test_models.py index b918f4c29f..7966524458 100644 --- a/selfdrive/car/tests/test_models.py +++ b/selfdrive/car/tests/test_models.py @@ -4,7 +4,6 @@ import pytest import random import unittest # noqa: TID251 from collections import defaultdict, Counter -from functools import partial import hypothesis.strategies as st from hypothesis import Phase, given, settings from parameterized import parameterized_class @@ -22,8 +21,7 @@ from openpilot.common.basedir import BASEDIR from openpilot.selfdrive.pandad import can_capnp_to_list from openpilot.selfdrive.test.helpers import read_segment_list from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT -from openpilot.tools.lib.logreader import LogReader, LogsUnavailable, openpilotci_source_zst, openpilotci_source, internal_source, \ - internal_source_zst, comma_api_source, auto_source +from openpilot.tools.lib.logreader import LogReader, LogsUnavailable, openpilotci_source, internal_source, comma_api_source from openpilot.tools.lib.route import SegmentName SafetyModel = car.CarParams.SafetyModel @@ -125,9 +123,8 @@ class TestCarModelBase(unittest.TestCase): segment_range = f"{cls.test_route.route}/{seg}" try: - source = partial(auto_source, sources=[internal_source, internal_source_zst] if len(INTERNAL_SEG_LIST) else \ - [openpilotci_source_zst, openpilotci_source, comma_api_source]) - lr = LogReader(segment_range, source=source, sort_by_time=True) + sources = [internal_source] if len(INTERNAL_SEG_LIST) else [openpilotci_source, comma_api_source] + lr = LogReader(segment_range, sources=sources, sort_by_time=True) return cls.get_testing_data_from_logreader(lr) except (LogsUnavailable, AssertionError): pass diff --git a/selfdrive/modeld/models/driving_policy.onnx b/selfdrive/modeld/models/driving_policy.onnx index 2582a9f1f5..ee9a9b6ec1 100644 --- a/selfdrive/modeld/models/driving_policy.onnx +++ b/selfdrive/modeld/models/driving_policy.onnx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cddefa5dbe21c60858f0d321d12d1ed4b126d05a7563f651bdf021674be63d64 +oid sha256:ef059460a95076f9a8600abb8c9d56c8c3b7384505b158064e2e875458b965bb size 15701037 diff --git a/system/ui/README.md b/system/ui/README.md index 85d32bfd6c..b124ae4d85 100644 --- a/system/ui/README.md +++ b/system/ui/README.md @@ -8,3 +8,8 @@ Quick start: * set `SCALE=1.5` to scale the entire UI by 1.5x * https://www.raylib.com/cheatsheet/cheatsheet.html * https://electronstudio.github.io/raylib-python-cffi/README.html#quickstart + +Style guide: +* All graphical elements should subclass [`Widget`](/system/ui/widgets/__init__.py). + * Prefer a stateful widget over a function for easy migration from QT +* All internal class variables and functions should be prefixed with `_` diff --git a/system/ui/reset.py b/system/ui/reset.py index aedca004b9..2178741ab5 100755 --- a/system/ui/reset.py +++ b/system/ui/reset.py @@ -1,14 +1,15 @@ #!/usr/bin/env python3 import os -import pyray as rl import sys import threading from enum import IntEnum +import pyray as rl + from openpilot.system.hardware import PC from openpilot.system.ui.lib.application import gui_app, FontWeight from openpilot.system.ui.widgets import Widget -from openpilot.system.ui.widgets.button import gui_button, ButtonStyle +from openpilot.system.ui.widgets.button import Button, ButtonStyle from openpilot.system.ui.widgets.label import gui_label, gui_text_box NVME = "/dev/nvme0n1" @@ -31,8 +32,15 @@ class ResetState(IntEnum): class Reset(Widget): def __init__(self, mode): super().__init__() - self.mode = mode - self.reset_state = ResetState.NONE + self._mode = mode + self._reset_state = ResetState.NONE + self._cancel_button = Button("Cancel", self._cancel_callback) + self._confirm_button = Button("Confirm", self._confirm, button_style=ButtonStyle.PRIMARY) + self._reboot_button = Button("Reboot", lambda: os.system("sudo reboot")) + self._render_status = True + + def _cancel_callback(self): + self._render_status = False def _do_erase(self): if PC: @@ -50,10 +58,10 @@ class Reset(Widget): if rm == 0 or fmt == 0: os.system("sudo reboot") else: - self.reset_state = ResetState.FAILED + self._reset_state = ResetState.FAILED def start_reset(self): - self.reset_state = ResetState.RESETTING + self._reset_state = ResetState.RESETTING threading.Timer(0.1, self._do_erase).start() def _render(self, rect: rl.Rectangle): @@ -61,42 +69,38 @@ class Reset(Widget): gui_label(label_rect, "System Reset", 100, font_weight=FontWeight.BOLD) text_rect = rl.Rectangle(rect.x + 140, rect.y + 140, rect.width - 280, rect.height - 90 - 100) - gui_text_box(text_rect, self.get_body_text(), 90) + gui_text_box(text_rect, self._get_body_text(), 90) button_height = 160 button_spacing = 50 button_top = rect.y + rect.height - button_height button_width = (rect.width - button_spacing) / 2.0 - if self.reset_state != ResetState.RESETTING: - if self.mode == ResetMode.RECOVER or self.reset_state == ResetState.FAILED: - if gui_button(rl.Rectangle(rect.x, button_top, button_width, button_height), "Reboot"): - os.system("sudo reboot") - elif self.mode == ResetMode.USER_RESET: - if gui_button(rl.Rectangle(rect.x, button_top, button_width, button_height), "Cancel"): - return False + if self._reset_state != ResetState.RESETTING: + if self._mode == ResetMode.RECOVER or self._reset_state == ResetState.FAILED: + self._reboot_button.render(rl.Rectangle(rect.x, button_top, rect.width, button_height)) + elif self._mode == ResetMode.USER_RESET: + self._cancel_button.render(rl.Rectangle(rect.x, button_top, button_width, button_height)) - if self.reset_state != ResetState.FAILED: - if gui_button(rl.Rectangle(rect.x + button_width + 50, button_top, button_width, button_height), - "Confirm", button_style=ButtonStyle.PRIMARY): - self.confirm() + if self._reset_state != ResetState.FAILED: + self._confirm_button.render(rl.Rectangle(rect.x + button_width + 50, button_top, button_width, button_height)) - return True + return self._render_status - def confirm(self): - if self.reset_state == ResetState.CONFIRM: + def _confirm(self): + if self._reset_state == ResetState.CONFIRM: self.start_reset() else: - self.reset_state = ResetState.CONFIRM + self._reset_state = ResetState.CONFIRM - def get_body_text(self): - if self.reset_state == ResetState.CONFIRM: + def _get_body_text(self): + if self._reset_state == ResetState.CONFIRM: return "Are you sure you want to reset your device?" - if self.reset_state == ResetState.RESETTING: + if self._reset_state == ResetState.RESETTING: return "Resetting device...\nThis may take up to a minute." - if self.reset_state == ResetState.FAILED: + if self._reset_state == ResetState.FAILED: return "Reset failed. Reboot to try again." - if self.mode == ResetMode.RECOVER: + if self._mode == ResetMode.RECOVER: return "Unable to mount data partition. Partition may be corrupted. Press confirm to erase and reset your device." return "System reset triggered. Press confirm to erase all content and settings. Press cancel to resume boot." diff --git a/system/ui/widgets/button.py b/system/ui/widgets/button.py index cfab842e93..2be56e6dd5 100644 --- a/system/ui/widgets/button.py +++ b/system/ui/widgets/button.py @@ -1,7 +1,11 @@ -import pyray as rl +from collections.abc import Callable from enum import IntEnum -from openpilot.system.ui.lib.application import gui_app, FontWeight + +import pyray as rl + +from openpilot.system.ui.lib.application import gui_app, FontWeight, MousePos from openpilot.system.ui.lib.text_measure import measure_text_cached +from openpilot.system.ui.widgets import Widget class ButtonStyle(IntEnum): @@ -148,3 +152,44 @@ def gui_button( rl.draw_text_ex(font, text, text_pos, font_size, 0, color) return result + + +class Button(Widget): + def __init__(self, + text: str, + click_callback: Callable[[], None] = None, + font_size: int = DEFAULT_BUTTON_FONT_SIZE, + font_weight: FontWeight = FontWeight.MEDIUM, + button_style: ButtonStyle = ButtonStyle.NORMAL, + border_radius: int = 10, + ): + + super().__init__() + self._text = text + self._click_callback = click_callback + self._label_font = gui_app.font(FontWeight.SEMI_BOLD) + self._button_style = button_style + self._font_size = font_size + self._border_radius = border_radius + self._font_size = font_size + self._text_color = BUTTON_TEXT_COLOR[button_style] + self._text_size = measure_text_cached(gui_app.font(font_weight), text, font_size) + + def _handle_mouse_release(self, mouse_pos: MousePos): + if self._click_callback: + print(f"Button clicked: {self._text}") + self._click_callback() + + def _get_background_color(self) -> rl.Color: + if self._is_pressed: + return BUTTON_PRESSED_BACKGROUND_COLORS[self._button_style] + else: + return BUTTON_BACKGROUND_COLORS[self._button_style] + + def _render(self, _): + roundness = self._border_radius / (min(self._rect.width, self._rect.height) / 2) + rl.draw_rectangle_rounded(self._rect, roundness, 10, self._get_background_color()) + + text_pos = rl.Vector2(0, self._rect.y + (self._rect.height - self._text_size.y) // 2) + text_pos.x = self._rect.x + (self._rect.width - self._text_size.x) // 2 + rl.draw_text_ex(self._label_font, self._text, text_pos, self._font_size, 0, self._text_color) diff --git a/tools/cabana/dbc/dbc.cc b/tools/cabana/dbc/dbc.cc index e9c869fce7..9b0de92218 100644 --- a/tools/cabana/dbc/dbc.cc +++ b/tools/cabana/dbc/dbc.cc @@ -109,7 +109,7 @@ void cabana::Msg::update() { mask[i] |= ((1ULL << sz) - 1) << shift; - bits -= size; + bits -= sz; i = sig->is_little_endian ? i - 1 : i + 1; } } diff --git a/tools/car_porting/examples/find_segments_with_message.ipynb b/tools/car_porting/examples/find_segments_with_message.ipynb index 4e688cc65b..af17bde52b 100644 --- a/tools/car_porting/examples/find_segments_with_message.ipynb +++ b/tools/car_porting/examples/find_segments_with_message.ipynb @@ -76,7 +76,7 @@ " if platform not in database:\n", " print(f\"No segments available for {platform}\")\n", " continue\n", - " \n", + "\n", " all_segments = database[platform]\n", " NUM_SEGMENTS = min(len(all_segments), MAX_SEGS_PER_PLATFORM)\n", " TEST_SEGMENTS.extend(random.sample(all_segments, NUM_SEGMENTS))\n", @@ -147,7 +147,7 @@ } ], "source": [ - "from openpilot.tools.lib.logreader import LogReader\n", + "from openpilot.tools.lib.logreader import LogReader, comma_car_segments_source\n", "from tqdm.notebook import tqdm, tnrange\n", "\n", "# Example search for CAN ignition messages\n", @@ -169,7 +169,7 @@ "progress_bar = tnrange(len(TEST_SEGMENTS), desc=\"segments searched\")\n", "\n", "for segment in TEST_SEGMENTS:\n", - " lr = LogReader(segment)\n", + " lr = LogReader(segment, sources=[comma_car_segments_source])\n", " CP = lr.first(\"carParams\")\n", " if CP is None:\n", " progress_bar.update()\n", diff --git a/tools/car_porting/examples/ford_vin_fingerprint.ipynb b/tools/car_porting/examples/ford_vin_fingerprint.ipynb index 7b0dd656da..6b806d22d2 100644 --- a/tools/car_porting/examples/ford_vin_fingerprint.ipynb +++ b/tools/car_porting/examples/ford_vin_fingerprint.ipynb @@ -20,7 +20,7 @@ "source": [ "\"\"\"In this example, we use the public comma car segments database to check if vin fingerprinting is feasible for ford.\"\"\"\n", "\n", - "from openpilot.tools.lib.logreader import LogReader\n", + "from openpilot.tools.lib.logreader import LogReader, comma_car_segments_source\n", "from openpilot.tools.lib.comma_car_segments import get_comma_car_segments_database\n", "from opendbc.car.ford.values import CAR\n", "\n", @@ -100,7 +100,7 @@ " if platform not in database:\n", " print(f\"Skipping platform: {platform}, no data available\")\n", " continue\n", - " \n", + "\n", " all_segments = database[platform]\n", "\n", " NUM_SEGMENTS = min(len(all_segments), MAX_SEGS_PER_PLATFORM)\n", @@ -110,7 +110,7 @@ " segments = random.sample(all_segments, NUM_SEGMENTS)\n", "\n", " for segment in segments:\n", - " lr = LogReader(segment)\n", + " lr = LogReader(segment, sources=[comma_car_segments_source])\n", " CP = lr.first(\"carParams\")\n", " if \"FORD\" not in CP.carFingerprint:\n", " print(segment, CP.carFingerprint)\n", diff --git a/tools/car_porting/examples/hkg_canfd_gear_message.ipynb b/tools/car_porting/examples/hkg_canfd_gear_message.ipynb index 5fdbdda684..f0bca8decc 100644 --- a/tools/car_porting/examples/hkg_canfd_gear_message.ipynb +++ b/tools/car_porting/examples/hkg_canfd_gear_message.ipynb @@ -72,7 +72,7 @@ " #if platform not in database:\n", " # print(f\"Skipping platform: {platform}, no data available\")\n", " # continue\n", - " \n", + "\n", " all_segments = database[platform]\n", "\n", " NUM_SEGMENTS = min(len(all_segments), MAX_SEGS_PER_PLATFORM)\n", @@ -198,12 +198,12 @@ "from opendbc.car.hyundai.hyundaicanfd import CanBus\n", "\n", "from openpilot.selfdrive.pandad import can_capnp_to_list\n", - "from openpilot.tools.lib.logreader import LogReader\n", + "from openpilot.tools.lib.logreader import LogReader, comma_car_segments_source\n", "\n", "message_names = [\"GEAR_SHIFTER\", \"ACCELERATOR\", \"GEAR\", \"GEAR_ALT\", \"GEAR_ALT_2\"]\n", "\n", "for segment in TEST_SEGMENTS:\n", - " lr = LogReader(segment)\n", + " lr = LogReader(segment, sources=[comma_car_segments_source])\n", " CP = lr.first(\"carParams\")\n", " if CP is None:\n", " continue\n", diff --git a/tools/lib/comma_car_segments.py b/tools/lib/comma_car_segments.py index 88496ae4d5..cd19356d66 100644 --- a/tools/lib/comma_car_segments.py +++ b/tools/lib/comma_car_segments.py @@ -14,7 +14,8 @@ def get_comma_car_segments_database(): ret = {} for platform in database: - ret[MIGRATION.get(platform, platform)] = database[platform] + # TODO: remove this when commaCarSegments is updated to remove selector + ret[MIGRATION.get(platform, platform)] = [s.rstrip('/s') for s in database[platform]] return ret diff --git a/tools/lib/filereader.py b/tools/lib/filereader.py index 0bb4abd2fa..02f5fd1b95 100644 --- a/tools/lib/filereader.py +++ b/tools/lib/filereader.py @@ -1,6 +1,8 @@ import os import posixpath import socket +from functools import cache +from openpilot.common.retry import retry from urllib.parse import urlparse from openpilot.tools.lib.url_file import URLFile @@ -8,14 +10,16 @@ from openpilot.tools.lib.url_file import URLFile DATA_ENDPOINT = os.getenv("DATA_ENDPOINT", "http://data-raw.comma.internal/") -def internal_source_available(url=DATA_ENDPOINT): +@cache +@retry(delay=0.0) +def internal_source_available(url: str) -> bool: if os.path.isdir(url): return True try: hostname = urlparse(url).hostname port = urlparse(url).port or 80 - with socket.socket(socket.AF_INET,socket.SOCK_STREAM) as s: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.settimeout(0.5) s.connect((hostname, port)) return True @@ -30,6 +34,7 @@ def resolve_name(fn): return fn +@cache def file_exists(fn): fn = resolve_name(fn) if fn.startswith(("http://", "https://")): diff --git a/tools/lib/helpers.py b/tools/lib/helpers.py index 7c34e17cb1..8c976e7ecc 100644 --- a/tools/lib/helpers.py +++ b/tools/lib/helpers.py @@ -9,7 +9,7 @@ class RE: INDEX = r'-?[0-9]+' SLICE = fr'(?P{INDEX})?:?(?P{INDEX})?:?(?P{INDEX})?' - SEGMENT_RANGE = fr'{ROUTE_NAME}(?:(--|/)(?P({SLICE})))?(?:/(?P([qras])))?' + SEGMENT_RANGE = fr'{ROUTE_NAME}(?:(--|/)(?P({SLICE})))?(?:/(?P([qra])))?' BOOTLOG_NAME = ROUTE_NAME diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index 34d3e5ea9f..473ba989b8 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 import bz2 -from functools import cache, partial +from functools import partial import multiprocessing import capnp import enum @@ -13,14 +13,15 @@ import warnings import zstandard as zstd from collections.abc import Callable, Iterable, Iterator +from typing import cast from urllib.parse import parse_qs, urlparse from cereal import log as capnp_log from openpilot.common.swaglog import cloudlog from openpilot.tools.lib.comma_car_segments import get_url as get_comma_segments_url from openpilot.tools.lib.openpilotci import get_url -from openpilot.tools.lib.filereader import FileReader, file_exists, internal_source_available -from openpilot.tools.lib.route import Route, SegmentRange +from openpilot.tools.lib.filereader import DATA_ENDPOINT, FileReader, file_exists, internal_source_available +from openpilot.tools.lib.route import QCAMERA_FILENAMES, CAMERA_FILENAMES, DCAMERA_FILENAMES, ECAMERA_FILENAMES, Route, SegmentRange from openpilot.tools.lib.log_time_series import msgs_to_time_series LogMessage = type[capnp._DynamicStructReader] @@ -96,14 +97,21 @@ class _LogFileReader: class ReadMode(enum.StrEnum): RLOG = "r" # only read rlogs QLOG = "q" # only read qlogs - SANITIZED = "s" # read from the commaCarSegments database AUTO = "a" # default to rlogs, fallback to qlogs AUTO_INTERACTIVE = "i" # default to rlogs, fallback to qlogs with a prompt from the user +class FileName(enum.Enum): + RLOG = ("rlog.zst", "rlog.bz2") + QLOG = ("qlog.zst", "qlog.bz2") + QCAMERA = tuple(QCAMERA_FILENAMES) + FCAMERA = tuple(CAMERA_FILENAMES) + ECAMERA = tuple(ECAMERA_FILENAMES) + DCAMERA = tuple(DCAMERA_FILENAMES) + + LogPath = str | None -ValidFileCallable = Callable[[LogPath], bool] -Source = Callable[[SegmentRange, ReadMode], list[LogPath]] +Source = Callable[[SegmentRange, FileName], list[LogPath]] InternalUnavailableException = Exception("Internal source not available") @@ -112,139 +120,129 @@ class LogsUnavailable(Exception): pass -@cache -def default_valid_file(fn: LogPath) -> bool: - return fn is not None and file_exists(fn) - - -def auto_strategy(rlog_paths: list[LogPath], qlog_paths: list[LogPath], interactive: bool, valid_file: ValidFileCallable) -> list[LogPath]: - # auto select logs based on availability - missing_rlogs = [rlog is None or not valid_file(rlog) for rlog in rlog_paths].count(True) - if missing_rlogs != 0: - if interactive: - if input(f"{missing_rlogs}/{len(rlog_paths)} rlogs were not found, would you like to fallback to qlogs for those segments? (y/n) ").lower() != "y": - return rlog_paths - else: - cloudlog.warning(f"{missing_rlogs}/{len(rlog_paths)} rlogs were not found, falling back to qlogs for those segments...") - - return [rlog if valid_file(rlog) else (qlog if valid_file(qlog) else None) - for (rlog, qlog) in zip(rlog_paths, qlog_paths, strict=True)] - return rlog_paths - - -def apply_strategy(mode: ReadMode, rlog_paths: list[LogPath], qlog_paths: list[LogPath], valid_file: ValidFileCallable = default_valid_file) -> list[LogPath]: - if mode == ReadMode.RLOG: - return rlog_paths - elif mode == ReadMode.QLOG: - return qlog_paths - elif mode == ReadMode.AUTO: - return auto_strategy(rlog_paths, qlog_paths, False, valid_file) - elif mode == ReadMode.AUTO_INTERACTIVE: - return auto_strategy(rlog_paths, qlog_paths, True, valid_file) - raise ValueError(f"invalid mode: {mode}") - - -def comma_api_source(sr: SegmentRange, mode: ReadMode) -> list[LogPath]: +def comma_api_source(sr: SegmentRange, fns: FileName) -> list[LogPath]: route = Route(sr.route_name) - rlog_paths = [route.log_paths()[seg] for seg in sr.seg_idxs] - qlog_paths = [route.qlog_paths()[seg] for seg in sr.seg_idxs] - # comma api will have already checked if the file exists - def valid_file(fn): - return fn is not None - - return apply_strategy(mode, rlog_paths, qlog_paths, valid_file=valid_file) + if fns == FileName.RLOG: + return [route.log_paths()[seg] for seg in sr.seg_idxs] + else: + return [route.qlog_paths()[seg] for seg in sr.seg_idxs] -def internal_source(sr: SegmentRange, mode: ReadMode, file_ext: str = "bz2") -> list[LogPath]: - if not internal_source_available(): +def internal_source(sr: SegmentRange, fns: FileName, endpoint_url: str = DATA_ENDPOINT) -> list[LogPath]: + if not internal_source_available(endpoint_url): raise InternalUnavailableException def get_internal_url(sr: SegmentRange, seg, file): - return f"cd:/{sr.dongle_id}/{sr.log_id}/{seg}/{file}.{file_ext}" + return f"{endpoint_url.rstrip('/')}/{sr.dongle_id}/{sr.log_id}/{seg}/{file}" - # TODO: list instead of using static URLs to support routes with multiple file extensions - rlog_paths = [get_internal_url(sr, seg, "rlog") for seg in sr.seg_idxs] - qlog_paths = [get_internal_url(sr, seg, "qlog") for seg in sr.seg_idxs] + return eval_source([[get_internal_url(sr, seg, fn) for fn in fns.value] for seg in sr.seg_idxs]) - return apply_strategy(mode, rlog_paths, qlog_paths) +def openpilotci_source(sr: SegmentRange, fns: FileName) -> list[LogPath]: + return eval_source([[get_url(sr.route_name, seg, fn) for fn in fns.value] for seg in sr.seg_idxs]) -def internal_source_zst(sr: SegmentRange, mode: ReadMode, file_ext: str = "zst") -> list[LogPath]: - return internal_source(sr, mode, file_ext) +def comma_car_segments_source(sr: SegmentRange, fns: FileName) -> list[LogPath]: + return eval_source([get_comma_segments_url(sr.route_name, seg) for seg in sr.seg_idxs]) -def openpilotci_source(sr: SegmentRange, mode: ReadMode, file_ext: str = "bz2") -> list[LogPath]: - rlog_paths = [get_url(sr.route_name, seg, f"rlog.{file_ext}") for seg in sr.seg_idxs] - qlog_paths = [get_url(sr.route_name, seg, f"qlog.{file_ext}") for seg in sr.seg_idxs] - return apply_strategy(mode, rlog_paths, qlog_paths) +def direct_source(file_or_url: str) -> list[str]: + return [file_or_url] -def openpilotci_source_zst(sr: SegmentRange, mode: ReadMode) -> list[LogPath]: - return openpilotci_source(sr, mode, "zst") +def eval_source(files: list[list[str] | str]) -> list[LogPath]: + # Returns valid file URLs given a list of possible file URLs for each segment (e.g. rlog.bz2, rlog.zst) + valid_files: list[LogPath] = [] + for urls in files: + if isinstance(urls, str): + urls = [urls] -def comma_car_segments_source(sr: SegmentRange, mode=ReadMode.RLOG) -> list[LogPath]: - return [get_comma_segments_url(sr.route_name, seg) for seg in sr.seg_idxs] + for url in urls: + if file_exists(url): + valid_files.append(url) + break + else: + valid_files.append(None) + return valid_files -def testing_closet_source(sr: SegmentRange, mode=ReadMode.RLOG) -> list[LogPath]: - if not internal_source_available('http://testing.comma.life'): - raise InternalUnavailableException - return [f"http://testing.comma.life/download/{sr.route_name.replace('|', '/')}/{seg}/rlog" for seg in sr.seg_idxs] +def auto_source(identifier: str, sources: list[Source], default_mode: ReadMode) -> list[str]: + exceptions = {} -def direct_source(file_or_url: str) -> list[LogPath]: - return [file_or_url] - + sr = SegmentRange(identifier) + mode = default_mode if sr.selector is None else ReadMode(sr.selector) -def get_invalid_files(files): - for f in files: - if f is None or not file_exists(f): - yield f + if mode == ReadMode.QLOG: + try_fns = [FileName.QLOG] + else: + try_fns = [FileName.RLOG] + # If selector allows it, fallback to qlogs + if mode in (ReadMode.AUTO, ReadMode.AUTO_INTERACTIVE): + try_fns.append(FileName.QLOG) -def check_source(source: Source, *args) -> list[LogPath]: - files = source(*args) - assert len(files) > 0, "No files on source" - assert next(get_invalid_files(files), False) is False, "Some files are invalid" - return files + # Build a dict of valid files as we evaluate each source. May contain mix of rlogs, qlogs, and None. + # This function only returns when we've sourced all files, or throws an exception + valid_files: dict[int, LogPath] = {} + for fn in try_fns: + for source in sources: + try: + files = source(sr, fn) + # Check every source returns an expected number of files + assert len(files) == len(valid_files) or len(valid_files) == 0, f"Source {source.__name__} returned unexpected number of files" -def auto_source(sr: SegmentRange, mode=ReadMode.RLOG, sources: list[Source] = None) -> list[LogPath]: - if mode == ReadMode.SANITIZED: - return comma_car_segments_source(sr, mode) + # Build a dict of valid files + for idx, f in enumerate(files): + if valid_files.get(idx) is None: + valid_files[idx] = f - if sources is None: - sources = [internal_source, internal_source_zst, openpilotci_source, openpilotci_source_zst, - comma_api_source, comma_car_segments_source, testing_closet_source] - exceptions = {} + # We've found all files, return them + if all(f is not None for f in valid_files.values()): + return cast(list[str], list(valid_files.values())) - # for automatic fallback modes, auto_source needs to first check if rlogs exist for any source - if mode in [ReadMode.AUTO, ReadMode.AUTO_INTERACTIVE]: - for source in sources: - try: - return check_source(source, sr, ReadMode.RLOG) - except Exception: - pass + except Exception as e: + exceptions[source.__name__] = e - # Automatically determine viable source - for source in sources: - try: - return check_source(source, sr, mode) - except Exception as e: - exceptions[source.__name__] = e + if fn == try_fns[0]: + missing_logs = list(valid_files.values()).count(None) + if mode == ReadMode.AUTO: + cloudlog.warning(f"{missing_logs}/{len(valid_files)} rlogs were not found, falling back to qlogs for those segments...") + elif mode == ReadMode.AUTO_INTERACTIVE: + if input(f"{missing_logs}/{len(valid_files)} rlogs were not found, would you like to fallback to qlogs for those segments? (y/N) ").lower() != "y": + break - raise LogsUnavailable("auto_source could not find any valid source, exceptions for sources:\n - " + - "\n - ".join([f"{k}: {repr(v)}" for k, v in exceptions.items()])) + missing_logs = list(valid_files.values()).count(None) + raise LogsUnavailable(f"{missing_logs}/{len(valid_files)} logs were not found, please ensure all logs " + + "are uploaded. You can fall back to qlogs with '/a' selector at the end of the route name.\n\n" + + "Exceptions for sources:\n - " + "\n - ".join([f"{k}: {repr(v)}" for k, v in exceptions.items()])) def parse_indirect(identifier: str) -> str: if "useradmin.comma.ai" in identifier: query = parse_qs(urlparse(identifier).query) - return query["onebox"][0] + identifier = query["onebox"][0] + elif "connect.comma.ai" in identifier: + path = urlparse(identifier).path.strip("/").split("/") + path = ['/'.join(path[:2]), *path[2:]] # recombine log id + + identifier = path[0] + if len(path) > 2: + # convert url with seconds to segments + start, end = int(path[1]) // 60, int(path[2]) // 60 + 1 + identifier = f"{identifier}/{start}:{end}" + + # add selector if it exists + if len(path) > 3: + identifier += f"/{path[3]}" + else: + # add selector if it exists + identifier = "/".join(path) + return identifier @@ -255,7 +253,7 @@ def parse_direct(identifier: str): class LogReader: - def _parse_identifier(self, identifier: str) -> list[LogPath]: + def _parse_identifier(self, identifier: str) -> list[str]: # useradmin, etc. identifier = parse_indirect(identifier) @@ -264,20 +262,16 @@ class LogReader: if direct_parsed is not None: return direct_source(identifier) - sr = SegmentRange(identifier) - mode = self.default_mode if sr.selector is None else ReadMode(sr.selector) - - identifiers = self.source(sr, mode) - - invalid_count = len(list(get_invalid_files(identifiers))) - assert invalid_count == 0, (f"{invalid_count}/{len(identifiers)} invalid log(s) found, please ensure all logs " + - "are uploaded or auto fallback to qlogs with '/a' selector at the end of the route name.") + identifiers = auto_source(identifier, self.sources, self.default_mode) return identifiers def __init__(self, identifier: str | list[str], default_mode: ReadMode = ReadMode.RLOG, - source: Source = auto_source, sort_by_time=False, only_union_types=False): + sources: list[Source] = None, sort_by_time=False, only_union_types=False): + if sources is None: + sources = [internal_source, openpilotci_source, comma_api_source, comma_car_segments_source] + self.default_mode = default_mode - self.source = source + self.sources = sources self.identifier = identifier if isinstance(identifier, str): self.identifier = [identifier] diff --git a/tools/lib/route.py b/tools/lib/route.py index 0a4700f083..5b5ccf3bc0 100644 --- a/tools/lib/route.py +++ b/tools/lib/route.py @@ -10,9 +10,9 @@ from openpilot.tools.lib.auth_config import get_token from openpilot.tools.lib.api import APIError, CommaApi from openpilot.tools.lib.helpers import RE -QLOG_FILENAMES = ['qlog', 'qlog.bz2', 'qlog.zst'] +QLOG_FILENAMES = ['qlog.bz2', 'qlog.zst', 'qlog'] QCAMERA_FILENAMES = ['qcamera.ts'] -LOG_FILENAMES = ['rlog', 'rlog.bz2', 'raw_log.bz2', 'rlog.zst'] +LOG_FILENAMES = ['rlog.bz2', 'raw_log.bz2', 'rlog.zst', 'rlog'] CAMERA_FILENAMES = ['fcamera.hevc', 'video.hevc'] DCAMERA_FILENAMES = ['dcamera.hevc'] ECAMERA_FILENAMES = ['ecamera.hevc'] @@ -241,6 +241,10 @@ class SegmentName: @property def canonical_name(self) -> str: return self._canonical_name + #TODO should only use one name + @property + def data_name(self) -> str: return f"{self._route_name.canonical_name}/{self._num}" + @property def dongle_id(self) -> str: return self._route_name.dongle_id diff --git a/tools/lib/tests/test_logreader.py b/tools/lib/tests/test_logreader.py index 230b6a65ea..11bdd33ccf 100644 --- a/tools/lib/tests/test_logreader.py +++ b/tools/lib/tests/test_logreader.py @@ -10,7 +10,7 @@ import requests from parameterized import parameterized from cereal import log as capnp_log -from openpilot.tools.lib.logreader import LogIterable, LogReader, comma_api_source, parse_indirect, ReadMode, InternalUnavailableException +from openpilot.tools.lib.logreader import LogsUnavailable, LogIterable, LogReader, comma_api_source, parse_indirect, ReadMode, InternalUnavailableException from openpilot.tools.lib.route import SegmentRange from openpilot.tools.lib.url_file import URLFileException @@ -193,17 +193,17 @@ class TestLogReader: with subtests.test("interactive_yes"): mocker.patch("sys.stdin", new=io.StringIO("y\n")) - lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, source=comma_api_source) + lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, sources=[comma_api_source]) log_len = len(list(lr)) assert qlog_len == log_len with subtests.test("interactive_no"): mocker.patch("sys.stdin", new=io.StringIO("n\n")) - with pytest.raises(AssertionError): - lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, source=comma_api_source) + with pytest.raises(LogsUnavailable): + lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, sources=[comma_api_source]) with subtests.test("non_interactive"): - lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO, source=comma_api_source) + lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO, sources=[comma_api_source]) log_len = len(list(lr)) assert qlog_len == log_len