diff --git a/tools/lib/helpers.py b/tools/lib/helpers.py index c184efe6a2..e2c8d07dd8 100644 --- a/tools/lib/helpers.py +++ b/tools/lib/helpers.py @@ -11,7 +11,7 @@ class RE: SEGMENT_NAME = r'{}(?:--|/)(?P[0-9]+)'.format(ROUTE_NAME) INDEX = r'-?[0-9]+' SLICE = r'(?P{})?:?(?P{})?:?(?P{})?'.format(INDEX, INDEX, INDEX) - SEGMENT_RANGE = r'{}(?:--|/)?(?P({}))?/?(?P([qr]))?'.format(ROUTE_NAME, SLICE) + SEGMENT_RANGE = r'{}(?:--|/)?(?P({}))?/?(?P([qra]))?'.format(ROUTE_NAME, SLICE) BOOTLOG_NAME = ROUTE_NAME EXPLORER_FILE = r'^(?P{})--(?P[a-z]+\.[a-z0-9]+)$'.format(SEGMENT_NAME) diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index c1f4f0f4a9..e28f41341c 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -16,6 +16,7 @@ from typing import Iterable, Iterator, List, Type from urllib.parse import parse_qs, urlparse from cereal import log as capnp_log +from openpilot.common.swaglog import cloudlog from openpilot.tools.lib.comma_car_segments import get_url as get_comma_segments_url from openpilot.tools.lib.openpilotci import get_url from openpilot.tools.lib.filereader import FileReader, file_exists @@ -71,8 +72,8 @@ class _LogFileReader: class ReadMode(enum.StrEnum): RLOG = "r" # only read rlogs QLOG = "q" # only read qlogs - #AUTO = "a" # default to rlogs, fallback to qlogs, not supported yet - + AUTO = "a" # default to rlogs, fallback to qlogs + AUTO_INTERACIVE = "i" # default to rlogs, fallback to qlogs with a prompt from the user def create_slice_from_string(s: str): m = re.fullmatch(RE.SLICE, s) @@ -86,31 +87,60 @@ def create_slice_from_string(s: str): return start return slice(start, end, step) +def auto_strategy(rlog_paths, qlog_paths, interactive): + # auto select logs based on availability + if any(rlog is None or not file_exists(rlog) for rlog in rlog_paths): + if interactive: + if input("Some rlogs were not found, would you like to fallback to qlogs for those segments? (y/n) ").lower() != "y": + return rlog_paths + else: + cloudlog.warning("Some rlogs were not found, falling back to qlogs for those segments...") + + return [rlog if (rlog is not None and file_exists(rlog)) else (qlog if (qlog is not None and file_exists(qlog)) else None) + for (rlog, qlog) in zip(rlog_paths, qlog_paths, strict=True)] + return rlog_paths + +def apply_strategy(mode: ReadMode, rlog_paths, qlog_paths): + if mode == ReadMode.RLOG: + return rlog_paths + elif mode == ReadMode.QLOG: + return qlog_paths + elif mode == ReadMode.AUTO: + return auto_strategy(rlog_paths, qlog_paths, False) + elif mode == ReadMode.AUTO_INTERACIVE: + return auto_strategy(rlog_paths, qlog_paths, True) + def parse_slice(sr: SegmentRange, route: Route): segs = np.arange(route.max_seg_number+1) s = create_slice_from_string(sr._slice) return segs[s] if isinstance(s, slice) else [segs[s]] -def comma_api_source(sr: SegmentRange, route: Route, mode=ReadMode.RLOG): +def comma_api_source(sr: SegmentRange, route: Route, mode: ReadMode): segs = parse_slice(sr, route) - log_paths = route.log_paths() if mode == ReadMode.RLOG else route.qlog_paths() + rlog_paths = [route.log_paths()[seg] for seg in segs] + qlog_paths = [route.log_paths()[seg] for seg in segs] - invalid_segs = [seg for seg in segs if log_paths[seg] is None] + return apply_strategy(mode, rlog_paths, qlog_paths) - assert not len(invalid_segs), f"Some of the requested segments are not available: {invalid_segs}" +def internal_source(sr: SegmentRange, route: Route, mode: ReadMode): + segs = parse_slice(sr, route) - return [(log_paths[seg]) for seg in segs] + def get_internal_url(sr: SegmentRange, seg, file): + return f"cd:/{sr.dongle_id}/{sr.timestamp}/{seg}/{file}.bz2" -def internal_source(sr: SegmentRange, route: Route, mode=ReadMode.RLOG): - segs = parse_slice(sr, route) + rlog_paths = [get_internal_url(sr, seg, "rlog") for seg in segs] + qlog_paths = [get_internal_url(sr, seg, "qlog") for seg in segs] - return [f"cd:/{sr.dongle_id}/{sr.timestamp}/{seg}/{'rlog' if mode == ReadMode.RLOG else 'qlog'}.bz2" for seg in segs] + return apply_strategy(mode, rlog_paths, qlog_paths) -def openpilotci_source(sr: SegmentRange, route: Route, mode=ReadMode.RLOG): +def openpilotci_source(sr: SegmentRange, route: Route, mode: ReadMode): segs = parse_slice(sr, route) - return [get_url(sr.route_name, seg, 'rlog' if mode == ReadMode.RLOG else 'qlog') for seg in segs] + rlog_paths = [get_url(sr.route_name, seg, "rlog") for seg in segs] + qlog_paths = [get_url(sr.route_name, seg, "qlog") for seg in segs] + + return apply_strategy(mode, rlog_paths, qlog_paths) def comma_car_segments_source(sr: SegmentRange, route: Route, mode=ReadMode.RLOG): segs = parse_slice(sr, route) @@ -120,10 +150,13 @@ def comma_car_segments_source(sr: SegmentRange, route: Route, mode=ReadMode.RLOG def direct_source(file_or_url): return [file_or_url] +def get_invalid_files(files): + return [f for f in files if f is None or not file_exists(f)] + def check_source(source, *args): try: files = source(*args) - assert all(file_exists(f) for f in files) + assert len(get_invalid_files(files)) == 0 return True, files except Exception: return False, None @@ -209,6 +242,9 @@ class LogReader: def reset(self): self.logreader_identifiers = self._parse_identifiers(self.identifier) + invalid_count = len(get_invalid_files(self.logreader_identifiers)) + assert invalid_count == 0, f"{invalid_count}/{len(self.logreader_identifiers)} invalid log(s) found, please ensure all logs \ +are uploaded or auto fallback to qlogs with '/a' selector at the end of the route name." @staticmethod def from_bytes(dat): diff --git a/tools/plotjuggler/juggle.py b/tools/plotjuggler/juggle.py index 292db5a50a..cc21095414 100755 --- a/tools/plotjuggler/juggle.py +++ b/tools/plotjuggler/juggle.py @@ -13,7 +13,7 @@ from functools import partial from openpilot.common.basedir import BASEDIR from openpilot.tools.lib.helpers import save_log -from openpilot.tools.lib.logreader import LogReader +from openpilot.tools.lib.logreader import LogReader, ReadMode juggle_dir = os.path.dirname(os.path.realpath(__file__)) @@ -73,7 +73,7 @@ def process(can, lr): return [d for d in lr if can or d.which() not in ['can', 'sendcan']] def juggle_route(route_or_segment_name, can, layout, dbc=None): - sr = LogReader(route_or_segment_name) + sr = LogReader(route_or_segment_name, default_mode=ReadMode.AUTO_INTERACIVE) all_data = sr.run_across_segments(24, partial(process, can))