From c8cc30112d9cd864c766e4600e759ef2c6fe2d70 Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Fri, 12 Jan 2024 15:53:50 -0500 Subject: [PATCH] rename segmentrangereader to logreader (#30981) * Replace multilogiterator * replace logreader * update readme * fix from_Bytes * new section * reset before iter * selector * fix internal * whitespace old-commit-hash: 3d73512c0702598d27e04039976fe0a91d5952f0 --- selfdrive/debug/can_print_changes.py | 7 +- selfdrive/debug/count_events.py | 4 +- selfdrive/debug/filter_log_message.py | 4 +- selfdrive/debug/fingerprint_from_route.py | 4 +- selfdrive/debug/run_process_on_route.py | 4 +- selfdrive/debug/toyota_eps_factor.py | 4 +- .../test/get_thumbnails_for_segment.py | 4 +- tools/car_porting/auto_fingerprint.py | 4 +- .../examples/subaru_long_accel.ipynb | 4 +- .../examples/subaru_steer_temp_fault.ipynb | 4 +- tools/latencylogger/latency_logger.py | 4 +- tools/lib/README.md | 20 +++ tools/lib/logreader.py | 153 +++++++++++++++++- tools/lib/srreader.py | 148 ----------------- .../{test_srreader.py => test_logreader.py} | 16 +- tools/plotjuggler/juggle.py | 4 +- tools/replay/can_replay.py | 4 +- tools/scripts/save_ubloxraw_stream.py | 4 +- tools/tuning/measure_steering_accuracy.py | 4 +- 19 files changed, 204 insertions(+), 196 deletions(-) delete mode 100644 tools/lib/srreader.py rename tools/lib/tests/{test_srreader.py => test_logreader.py} (83%) diff --git a/selfdrive/debug/can_print_changes.py b/selfdrive/debug/can_print_changes.py index 20fe502efc..810e45cfc6 100755 --- a/selfdrive/debug/can_print_changes.py +++ b/selfdrive/debug/can_print_changes.py @@ -7,8 +7,7 @@ from typing import Optional import cereal.messaging as messaging from openpilot.selfdrive.debug.can_table import can_table -from openpilot.tools.lib.logreader import LogIterable -from openpilot.tools.lib.srreader import SegmentRangeReader +from openpilot.tools.lib.logreader import LogIterable, LogReader RED = '\033[91m' CLEAR = '\033[0m' @@ -104,8 +103,8 @@ if __name__ == "__main__": if args.init == '': init_lr = [] else: - init_lr = SegmentRangeReader(args.init) + init_lr = LogReader(args.init) if args.comp: - new_lr = SegmentRangeReader(args.comp) + new_lr = LogReader(args.comp) can_printer(args.bus, init_msgs=init_lr, new_msgs=new_lr, table=args.table) diff --git a/selfdrive/debug/count_events.py b/selfdrive/debug/count_events.py index 70f412bff1..0d545b3153 100755 --- a/selfdrive/debug/count_events.py +++ b/selfdrive/debug/count_events.py @@ -7,7 +7,7 @@ from pprint import pprint from typing import List, Tuple, cast from cereal.services import SERVICE_LIST -from openpilot.tools.lib.srreader import SegmentRangeReader, ReadMode +from openpilot.tools.lib.logreader import LogReader, ReadMode if __name__ == "__main__": cnt_valid: Counter = Counter() @@ -20,7 +20,7 @@ if __name__ == "__main__": start_time = math.inf end_time = -math.inf ignition_off = None - for msg in SegmentRangeReader(sys.argv[1], ReadMode.QLOG): + for msg in LogReader(sys.argv[1], ReadMode.QLOG): end_time = max(end_time, msg.logMonoTime) start_time = min(start_time, msg.logMonoTime) diff --git a/selfdrive/debug/filter_log_message.py b/selfdrive/debug/filter_log_message.py index f8e78c8b98..20028f8fd2 100755 --- a/selfdrive/debug/filter_log_message.py +++ b/selfdrive/debug/filter_log_message.py @@ -3,7 +3,7 @@ import argparse import json import cereal.messaging as messaging -from openpilot.tools.lib.srreader import SegmentRangeReader +from openpilot.tools.lib.logreader import LogReader LEVELS = { "DEBUG": 10, @@ -55,7 +55,7 @@ if __name__ == "__main__": if args.route: for route in args.route: - lr = SegmentRangeReader(route) + lr = LogReader(route) for m in lr: if m.which() == 'logMessage': print_logmessage(m.logMonoTime, m.logMessage, min_level) diff --git a/selfdrive/debug/fingerprint_from_route.py b/selfdrive/debug/fingerprint_from_route.py index 7126a282c9..68308bb627 100755 --- a/selfdrive/debug/fingerprint_from_route.py +++ b/selfdrive/debug/fingerprint_from_route.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import sys -from openpilot.tools.lib.srreader import ReadMode, SegmentRangeReader +from openpilot.tools.lib.logreader import LogReader, ReadMode def get_fingerprint(lr): @@ -39,5 +39,5 @@ if __name__ == "__main__": print("Usage: ./fingerprint_from_route.py ") sys.exit(1) - lr = SegmentRangeReader(sys.argv[1], ReadMode.QLOG) + lr = LogReader(sys.argv[1], ReadMode.QLOG) get_fingerprint(lr) diff --git a/selfdrive/debug/run_process_on_route.py b/selfdrive/debug/run_process_on_route.py index 51ba06ec14..441de9ef57 100755 --- a/selfdrive/debug/run_process_on_route.py +++ b/selfdrive/debug/run_process_on_route.py @@ -4,7 +4,7 @@ import argparse from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, replay_process from openpilot.tools.lib.helpers import save_log -from openpilot.tools.lib.srreader import SegmentRangeReader +from openpilot.tools.lib.logreader import LogReader if __name__ == "__main__": parser = argparse.ArgumentParser(description="Run process on route and create new logs", @@ -16,7 +16,7 @@ if __name__ == "__main__": cfg = [c for c in CONFIGS if c.proc_name == args.process][0] - lr = SegmentRangeReader(args.route) + lr = LogReader(args.route) inputs = list(lr) outputs = replay_process(cfg, inputs, fingerprint=args.fingerprint) diff --git a/selfdrive/debug/toyota_eps_factor.py b/selfdrive/debug/toyota_eps_factor.py index 1f72fa296a..dc83c8f7a4 100755 --- a/selfdrive/debug/toyota_eps_factor.py +++ b/selfdrive/debug/toyota_eps_factor.py @@ -5,7 +5,7 @@ import matplotlib.pyplot as plt from sklearn import linear_model from openpilot.selfdrive.car.toyota.values import STEER_THRESHOLD -from openpilot.tools.lib.srreader import SegmentRangeReader +from openpilot.tools.lib.logreader import LogReader MIN_SAMPLES = 30 * 100 @@ -57,6 +57,6 @@ def get_eps_factor(lr, plot=False): if __name__ == "__main__": - lr = SegmentRangeReader(sys.argv[1]) + lr = LogReader(sys.argv[1]) n = get_eps_factor(lr, plot="--plot" in sys.argv) print("EPS torque factor: ", n) diff --git a/system/camerad/test/get_thumbnails_for_segment.py b/system/camerad/test/get_thumbnails_for_segment.py index 95d865e4e4..21409f398d 100755 --- a/system/camerad/test/get_thumbnails_for_segment.py +++ b/system/camerad/test/get_thumbnails_for_segment.py @@ -3,7 +3,7 @@ import argparse import os from tqdm import tqdm -from openpilot.tools.lib.srreader import SegmentRangeReader +from openpilot.tools.lib.logreader import LogReader if __name__ == "__main__": parser = argparse.ArgumentParser() @@ -13,7 +13,7 @@ if __name__ == "__main__": out_path = os.path.join("jpegs", f"{args.route.replace('|', '_').replace('/', '_')}") os.makedirs(out_path, exist_ok=True) - lr = SegmentRangeReader(args.route) + lr = LogReader(args.route) for msg in tqdm(lr): if msg.which() == 'thumbnail': diff --git a/tools/car_porting/auto_fingerprint.py b/tools/car_porting/auto_fingerprint.py index ef3efc1cf3..fce7e33182 100755 --- a/tools/car_porting/auto_fingerprint.py +++ b/tools/car_porting/auto_fingerprint.py @@ -7,7 +7,7 @@ from openpilot.selfdrive.debug.format_fingerprints import format_brand_fw_versio from openpilot.selfdrive.car.fw_versions import match_fw_to_car from openpilot.selfdrive.car.interfaces import get_interface_attr -from openpilot.tools.lib.srreader import SegmentRangeReader, ReadMode +from openpilot.tools.lib.logreader import LogReader, ReadMode ALL_FW_VERSIONS = get_interface_attr("FW_VERSIONS") @@ -24,7 +24,7 @@ if __name__ == "__main__": parser.add_argument("platform", help="The platform, or leave empty to auto-determine using fuzzy", default=None, nargs='?') args = parser.parse_args() - lr = SegmentRangeReader(args.route, ReadMode.QLOG) + lr = LogReader(args.route, ReadMode.QLOG) carFw = None carVin = None diff --git a/tools/car_porting/examples/subaru_long_accel.ipynb b/tools/car_porting/examples/subaru_long_accel.ipynb index ea685dcd4f..24470b44d2 100644 --- a/tools/car_porting/examples/subaru_long_accel.ipynb +++ b/tools/car_porting/examples/subaru_long_accel.ipynb @@ -24,14 +24,14 @@ "from opendbc.can.parser import CANParser\n", "\n", "from openpilot.selfdrive.car.subaru.values import DBC\n", - "from openpilot.tools.lib.srreader import SegmentRangeReader\n", + "from openpilot.tools.lib.logreader import LogReader\n", "\n", "\"\"\"\n", "In this example, we plot the relationship between Cruise_Brake and Acceleration for stock eyesight.\n", "\"\"\"\n", "\n", "for segment in segments:\n", - " lr = SegmentRangeReader(segment)\n", + " lr = LogReader(segment)\n", "\n", " messages = [\n", " (\"ES_Distance\", 20),\n", diff --git a/tools/car_porting/examples/subaru_steer_temp_fault.ipynb b/tools/car_porting/examples/subaru_steer_temp_fault.ipynb index b60915e6ec..46d8dc413e 100644 --- a/tools/car_porting/examples/subaru_steer_temp_fault.ipynb +++ b/tools/car_porting/examples/subaru_steer_temp_fault.ipynb @@ -27,7 +27,7 @@ "from opendbc.can.parser import CANParser\n", "\n", "from openpilot.selfdrive.car.subaru.values import CanBus, DBC\n", - "from openpilot.tools.lib.srreader import SegmentRangeReader\n", + "from openpilot.tools.lib.logreader import LogReader\n", "\n", "\"\"\"\n", "In this example, we search for positive transitions of Steer_Warning, which indicate that the EPS\n", @@ -36,7 +36,7 @@ "\"\"\"\n", "\n", "for segment in segments:\n", - " lr = SegmentRangeReader(segment)\n", + " lr = LogReader(segment)\n", "\n", " can_msgs = [msg for msg in lr if msg.which() == \"can\"]\n", "\n", diff --git a/tools/latencylogger/latency_logger.py b/tools/latencylogger/latency_logger.py index 29d4889d35..b561b65105 100755 --- a/tools/latencylogger/latency_logger.py +++ b/tools/latencylogger/latency_logger.py @@ -8,7 +8,7 @@ import sys from bisect import bisect_left, bisect_right from collections import defaultdict -from openpilot.tools.lib.srreader import SegmentRangeReader +from openpilot.tools.lib.logreader import LogReader DEMO_ROUTE = "9f583b1d93915c31|2022-05-18--10-49-51--0" @@ -236,7 +236,7 @@ if __name__ == "__main__": args = parser.parse_args() r = DEMO_ROUTE if args.demo else args.route_or_segment_name.strip() - lr = SegmentRangeReader(r, sort_by_time=True) + lr = LogReader(r, sort_by_time=True) data, _ = get_timestamps(lr) print_timestamps(data['timestamp'], data['duration'], data['start'], args.relative) diff --git a/tools/lib/README.md b/tools/lib/README.md index 7d8f2f8bf9..a0c4a0863b 100644 --- a/tools/lib/README.md +++ b/tools/lib/README.md @@ -31,3 +31,23 @@ for msg in lr: if msg.which() == "carState": print(msg.carState.steeringAngleDeg) ``` + +### Segment Ranges + +We also support a new format called a "segment range", where you can specify which segments from a route to load. + +```python + +lr = LogReader("a2a0ccea32023010|2023-07-27--13-01-19/4") # 4th segment +lr = LogReader("a2a0ccea32023010|2023-07-27--13-01-19/4:6") # 4th and 5th segment +lr = LogReader("a2a0ccea32023010|2023-07-27--13-01-19/-1") # last segment +lr = LogReader("a2a0ccea32023010|2023-07-27--13-01-19/:5") # first 5 segments +lr = LogReader("a2a0ccea32023010|2023-07-27--13-01-19/1:") # all except first segment +``` + +and can select which type of logs to grab + +```python +lr = LogReader("a2a0ccea32023010|2023-07-27--13-01-19/4/q") # get qlogs +lr = LogReader("a2a0ccea32023010|2023-07-27--13-01-19/4/r") # get rlogs (default) +``` diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index 924e07afe9..7142c7f854 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -1,20 +1,29 @@ #!/usr/bin/env python3 +import bz2 +import capnp +import enum +import itertools +import numpy as np import os +import pathlib +import re import sys -import bz2 import urllib.parse -import capnp import warnings from typing import Iterable, Iterator +from urllib.parse import parse_qs, urlparse from cereal import log as capnp_log +from openpilot.selfdrive.test.openpilotci import get_url from openpilot.tools.lib.filereader import FileReader +from openpilot.tools.lib.helpers import RE +from openpilot.tools.lib.route import Route, SegmentRange LogIterable = Iterable[capnp._DynamicStructReader] -class LogReader: +class _LogFileReader: def __init__(self, fn, canonicalize=True, only_union_types=False, sort_by_time=False, dat=None): self.data_version = None self._only_union_types = only_union_types @@ -44,10 +53,6 @@ class LogReader: self._ents = list(sorted(_ents, key=lambda x: x.logMonoTime) if sort_by_time else _ents) self._ts = [x.logMonoTime for x in self._ents] - @classmethod - def from_bytes(cls, dat): - return cls("", dat=dat) - def __iter__(self) -> Iterator[capnp._DynamicStructReader]: for ent in self._ents: if self._only_union_types: @@ -60,6 +65,140 @@ class LogReader: yield ent +class ReadMode(enum.StrEnum): + RLOG = "r" # only read rlogs + QLOG = "q" # only read qlogs + #AUTO = "a" # default to rlogs, fallback to qlogs, not supported yet + + +def create_slice_from_string(s: str): + m = re.fullmatch(RE.SLICE, s) + assert m is not None, f"Invalid slice: {s}" + start, end, step = m.groups() + start = int(start) if start is not None else None + end = int(end) if end is not None else None + step = int(step) if step is not None else None + + if start is not None and ":" not in s and end is None and step is None: + return start + return slice(start, end, step) + +def parse_slice(sr: SegmentRange): + route = Route(sr.route_name) + segs = np.arange(route.max_seg_number+1) + s = create_slice_from_string(sr._slice) + return segs[s] if isinstance(s, slice) else [segs[s]] + +def comma_api_source(sr: SegmentRange, mode=ReadMode.RLOG, sort_by_time=False): + segs = parse_slice(sr) + route = Route(sr.route_name) + + log_paths = route.log_paths() if mode == ReadMode.RLOG else route.qlog_paths() + + invalid_segs = [seg for seg in segs if log_paths[seg] is None] + + assert not len(invalid_segs), f"Some of the requested segments are not available: {invalid_segs}" + + for seg in segs: + yield _LogFileReader(log_paths[seg], sort_by_time=sort_by_time) + +def internal_source(sr: SegmentRange, mode=ReadMode.RLOG, sort_by_time=False): + segs = parse_slice(sr) + + for seg in segs: + yield _LogFileReader(f"cd:/{sr.dongle_id}/{sr.timestamp}/{seg}/{'rlog' if mode == ReadMode.RLOG else 'qlog'}.bz2", sort_by_time=sort_by_time) + +def openpilotci_source(sr: SegmentRange, mode=ReadMode.RLOG, sort_by_time=False): + segs = parse_slice(sr) + + for seg in segs: + yield _LogFileReader(get_url(sr.route_name, seg, 'rlog' if mode == ReadMode.RLOG else 'qlog'), sort_by_time=sort_by_time) + +def direct_source(file_or_url, sort_by_time): + yield _LogFileReader(file_or_url, sort_by_time=sort_by_time) + +def auto_source(*args, **kwargs): + # Automatically determine viable source + + try: + next(internal_source(*args, **kwargs)) + return internal_source(*args, **kwargs) + except Exception: + pass + + try: + next(openpilotci_source(*args, **kwargs)) + return openpilotci_source(*args, **kwargs) + except Exception: + pass + + return comma_api_source(*args, **kwargs) + +def parse_useradmin(identifier): + if "useradmin.comma.ai" in identifier: + query = parse_qs(urlparse(identifier).query) + return query["onebox"][0] + return None + +def parse_cabana(identifier): + if "cabana.comma.ai" in identifier: + query = parse_qs(urlparse(identifier).query) + return query["route"][0] + return None + +def parse_direct(identifier): + if identifier.startswith(("http://", "https://", "cd:/")) or pathlib.Path(identifier).exists(): + return identifier + return None + +def parse_indirect(identifier): + parsed = parse_useradmin(identifier) or parse_cabana(identifier) + + if parsed is not None: + return parsed, comma_api_source, True + + return identifier, None, False + + +class LogReader: + def _logreaders_from_identifier(self, identifier): + parsed, source, is_indirect = parse_indirect(identifier) + + if not is_indirect: + direct_parsed = parse_direct(identifier) + if direct_parsed is not None: + return direct_source(identifier, sort_by_time=self.sort_by_time) + + sr = SegmentRange(parsed) + mode = self.default_mode if sr.selector is None else ReadMode(sr.selector) + source = self.default_source if source is None else source + + return source(sr, mode, sort_by_time=self.sort_by_time) + + def __init__(self, identifier: str, default_mode=ReadMode.RLOG, default_source=auto_source, sort_by_time=False): + self.default_mode = default_mode + self.default_source = default_source + self.sort_by_time = sort_by_time + self.identifier = identifier + + self.reset() + + def __iter__(self): + self.reset() + return self + + def __next__(self): + return next(self.chain) + + def reset(self): + self.lrs = self._logreaders_from_identifier(self.identifier) + self.chain = itertools.chain(*self.lrs) + + @staticmethod + def from_bytes(dat): + return _LogFileReader("", dat=dat) + + if __name__ == "__main__": import codecs # capnproto <= 0.8.0 throws errors converting byte data to string diff --git a/tools/lib/srreader.py b/tools/lib/srreader.py deleted file mode 100644 index ea11e5e3c5..0000000000 --- a/tools/lib/srreader.py +++ /dev/null @@ -1,148 +0,0 @@ -import enum -import itertools -import numpy as np -import pathlib -import re -from urllib.parse import parse_qs, urlparse - -from openpilot.selfdrive.test.openpilotci import get_url -from openpilot.tools.lib.helpers import RE -from openpilot.tools.lib.logreader import LogReader -from openpilot.tools.lib.route import Route, SegmentRange - -class ReadMode(enum.StrEnum): - RLOG = "r" # only read rlogs - QLOG = "q" # only read qlogs - #AUTO = "a" # default to rlogs, fallback to qlogs, not supported yet - - -def create_slice_from_string(s: str): - m = re.fullmatch(RE.SLICE, s) - assert m is not None, f"Invalid slice: {s}" - start, end, step = m.groups() - start = int(start) if start is not None else None - end = int(end) if end is not None else None - step = int(step) if step is not None else None - - if start is not None and ":" not in s and end is None and step is None: - return start - return slice(start, end, step) - - -def parse_slice(sr: SegmentRange): - route = Route(sr.route_name) - segs = np.arange(route.max_seg_number+1) - s = create_slice_from_string(sr._slice) - return segs[s] if isinstance(s, slice) else [segs[s]] - -def comma_api_source(sr: SegmentRange, mode=ReadMode.RLOG, sort_by_time=False): - segs = parse_slice(sr) - route = Route(sr.route_name) - - log_paths = route.log_paths() if mode == ReadMode.RLOG else route.qlog_paths() - - invalid_segs = [seg for seg in segs if log_paths[seg] is None] - - assert not len(invalid_segs), f"Some of the requested segments are not available: {invalid_segs}" - - for seg in segs: - yield LogReader(log_paths[seg], sort_by_time=sort_by_time) - -def internal_source(sr: SegmentRange, mode=ReadMode.RLOG, sort_by_time=False): - segs = parse_slice(sr) - - for seg in segs: - yield LogReader(f"cd:/{sr.dongle_id}/{sr.timestamp}/{seg}/{'rlog' if mode == ReadMode.RLOG else 'qlog'}.bz2", sort_by_time=sort_by_time) - -def openpilotci_source(sr: SegmentRange, mode=ReadMode.RLOG, sort_by_time=False): - segs = parse_slice(sr) - - for seg in segs: - yield LogReader(get_url(sr.route_name, seg, 'rlog' if mode == ReadMode.RLOG else 'qlog'), sort_by_time=sort_by_time) - -def direct_source(file_or_url, sort_by_time): - yield LogReader(file_or_url, sort_by_time=sort_by_time) - -def auto_source(*args, **kwargs): - # Automatically determine viable source - - try: - next(internal_source(*args, **kwargs)) - return internal_source(*args, **kwargs) - except Exception: - pass - - try: - next(openpilotci_source(*args, **kwargs)) - return openpilotci_source(*args, **kwargs) - except Exception: - pass - - return comma_api_source(*args, **kwargs) - -def parse_useradmin(identifier): - if "useradmin.comma.ai" in identifier: - query = parse_qs(urlparse(identifier).query) - return query["onebox"][0] - return None - -def parse_cabana(identifier): - if "cabana.comma.ai" in identifier: - query = parse_qs(urlparse(identifier).query) - return query["route"][0] - return None - -def parse_cd(identifier): - if "cd:/" in identifier: - return identifier.replace("cd:/", "") - return None - -def parse_direct(identifier): - if "https://" in identifier or "http://" in identifier or pathlib.Path(identifier).exists(): - return identifier - return None - -def parse_indirect(identifier): - parsed = parse_useradmin(identifier) or parse_cabana(identifier) - - if parsed is not None: - return parsed, comma_api_source, True - - parsed = parse_cd(identifier) - if parsed is not None: - return parsed, internal_source, True - - return identifier, None, False - -class SegmentRangeReader: - def _logreaders_from_identifier(self, identifier): - parsed, source, is_indirect = parse_indirect(identifier) - - if not is_indirect: - direct_parsed = parse_direct(identifier) - if direct_parsed is not None: - return direct_source(identifier, sort_by_time=self.sort_by_time) - - sr = SegmentRange(parsed) - mode = self.default_mode if sr.selector is None else ReadMode(sr.selector) - source = self.default_source if source is None else source - - return source(sr, mode, sort_by_time=self.sort_by_time) - - def __init__(self, identifier: str, default_mode=ReadMode.RLOG, default_source=auto_source, sort_by_time=False): - self.default_mode = default_mode - self.default_source = default_source - self.sort_by_time = sort_by_time - self.identifier = identifier - - self.reset() - - def __iter__(self): - return self - - def __next__(self): - return next(self.chain) - - def reset(self): - self.lrs = self._logreaders_from_identifier(self.identifier) - self.chain = itertools.chain(*self.lrs) diff --git a/tools/lib/tests/test_srreader.py b/tools/lib/tests/test_logreader.py similarity index 83% rename from tools/lib/tests/test_srreader.py rename to tools/lib/tests/test_logreader.py index b22599dee8..db207a6522 100644 --- a/tools/lib/tests/test_srreader.py +++ b/tools/lib/tests/test_logreader.py @@ -4,16 +4,15 @@ import numpy as np import unittest from parameterized import parameterized import requests - +from openpilot.tools.lib.logreader import LogReader, parse_indirect, parse_slice, ReadMode from openpilot.tools.lib.route import SegmentRange -from openpilot.tools.lib.srreader import ReadMode, SegmentRangeReader, parse_slice, parse_indirect NUM_SEGS = 17 # number of segments in the test route ALL_SEGS = list(np.arange(NUM_SEGS)) TEST_ROUTE = "344c5c15b34f2d8a/2024-01-03--09-37-12" QLOG_FILE = "https://commadataci.blob.core.windows.net/openpilotci/0375fdf7b1ce594d/2019-06-13--08-32-25/3/qlog.bz2" -class TestSegmentRangeReader(unittest.TestCase): +class TestLogReader(unittest.TestCase): @parameterized.expand([ (f"{TEST_ROUTE}", ALL_SEGS), (f"{TEST_ROUTE.replace('/', '|')}", ALL_SEGS), @@ -38,7 +37,6 @@ class TestSegmentRangeReader(unittest.TestCase): (f"https://useradmin.comma.ai/?onebox={TEST_ROUTE.replace('/', '|')}", ALL_SEGS), (f"https://useradmin.comma.ai/?onebox={TEST_ROUTE.replace('/', '%7C')}", ALL_SEGS), (f"https://cabana.comma.ai/?route={TEST_ROUTE}", ALL_SEGS), - (f"cd:/{TEST_ROUTE}", ALL_SEGS), ]) def test_indirect_parsing(self, identifier, expected): parsed, _, _ = parse_indirect(identifier) @@ -54,7 +52,7 @@ class TestSegmentRangeReader(unittest.TestCase): shutil.copyfileobj(r.raw, f) for f in [QLOG_FILE, qlog.name]: - l = len(list(SegmentRangeReader(f))) + l = len(list(LogReader(f))) self.assertGreater(l, 100) @parameterized.expand([ @@ -72,14 +70,14 @@ class TestSegmentRangeReader(unittest.TestCase): parse_slice(sr) def test_modes(self): - qlog_len = len(list(SegmentRangeReader(f"{TEST_ROUTE}/0", ReadMode.QLOG))) - rlog_len = len(list(SegmentRangeReader(f"{TEST_ROUTE}/0", ReadMode.RLOG))) + qlog_len = len(list(LogReader(f"{TEST_ROUTE}/0", ReadMode.QLOG))) + rlog_len = len(list(LogReader(f"{TEST_ROUTE}/0", ReadMode.RLOG))) self.assertLess(qlog_len * 6, rlog_len) def test_modes_from_name(self): - qlog_len = len(list(SegmentRangeReader(f"{TEST_ROUTE}/0/q"))) - rlog_len = len(list(SegmentRangeReader(f"{TEST_ROUTE}/0/r"))) + qlog_len = len(list(LogReader(f"{TEST_ROUTE}/0/q"))) + rlog_len = len(list(LogReader(f"{TEST_ROUTE}/0/r"))) self.assertLess(qlog_len * 6, rlog_len) diff --git a/tools/plotjuggler/juggle.py b/tools/plotjuggler/juggle.py index 3df777c959..b497bdaa39 100755 --- a/tools/plotjuggler/juggle.py +++ b/tools/plotjuggler/juggle.py @@ -14,7 +14,7 @@ from functools import partial from openpilot.common.basedir import BASEDIR from openpilot.tools.lib.helpers import save_log -from openpilot.tools.lib.srreader import SegmentRangeReader +from openpilot.tools.lib.logreader import LogReader juggle_dir = os.path.dirname(os.path.realpath(__file__)) @@ -74,7 +74,7 @@ def process(can, lr): return [d for d in lr if can or d.which() not in ['can', 'sendcan']] def juggle_route(route_or_segment_name, can, layout, dbc=None): - sr = SegmentRangeReader(route_or_segment_name) + sr = LogReader(route_or_segment_name) with multiprocessing.Pool(24) as pool: all_data = [] diff --git a/tools/replay/can_replay.py b/tools/replay/can_replay.py index 4be516a108..d0a5304cff 100755 --- a/tools/replay/can_replay.py +++ b/tools/replay/can_replay.py @@ -9,7 +9,7 @@ os.environ['FILEREADER_CACHE'] = '1' from openpilot.common.realtime import config_realtime_process, Ratekeeper, DT_CTRL from openpilot.selfdrive.boardd.boardd import can_capnp_to_can_list -from openpilot.tools.lib.srreader import SegmentRangeReader +from openpilot.tools.lib.logreader import LogReader from panda import Panda, PandaJungle def send_thread(s, flock): @@ -97,7 +97,7 @@ if __name__ == "__main__": if args.route_or_segment_name is None: args.route_or_segment_name = "77611a1fac303767/2020-03-24--09-50-38/10:16" - sr = SegmentRangeReader(args.route_or_segment_name) + sr = LogReader(args.route_or_segment_name) with multiprocessing.Pool(24) as pool: CAN_MSGS = [] diff --git a/tools/scripts/save_ubloxraw_stream.py b/tools/scripts/save_ubloxraw_stream.py index 39722b4f29..ecbc2bb31e 100755 --- a/tools/scripts/save_ubloxraw_stream.py +++ b/tools/scripts/save_ubloxraw_stream.py @@ -3,7 +3,7 @@ import argparse import os import sys from openpilot.common.basedir import BASEDIR -from openpilot.tools.lib.srreader import SegmentRangeReader +from openpilot.tools.lib.logreader import LogReader os.environ['BASEDIR'] = BASEDIR @@ -23,7 +23,7 @@ def get_arg_parser(): def main(): args = get_arg_parser().parse_args(sys.argv[1:]) - lr = SegmentRangeReader(args.route) + lr = LogReader(args.route) with open(args.out_path, 'wb') as f: try: diff --git a/tools/tuning/measure_steering_accuracy.py b/tools/tuning/measure_steering_accuracy.py index de4de46c2a..06523497a5 100755 --- a/tools/tuning/measure_steering_accuracy.py +++ b/tools/tuning/measure_steering_accuracy.py @@ -8,7 +8,7 @@ import signal from collections import defaultdict import cereal.messaging as messaging -from openpilot.tools.lib.srreader import SegmentRangeReader +from openpilot.tools.lib.logreader import LogReader def sigint_handler(signal, frame): exit(0) @@ -128,7 +128,7 @@ if __name__ == "__main__": if args.route is not None: print(f"loading {args.route}...") - lr = SegmentRangeReader(args.route, sort_by_time=True) + lr = LogReader(args.route, sort_by_time=True) sm = {} for msg in lr: