|
|
|
@ -16,6 +16,7 @@ from typing import Iterable, Iterator, List, Type |
|
|
|
from urllib.parse import parse_qs, urlparse |
|
|
|
from urllib.parse import parse_qs, urlparse |
|
|
|
|
|
|
|
|
|
|
|
from cereal import log as capnp_log |
|
|
|
from cereal import log as capnp_log |
|
|
|
|
|
|
|
from openpilot.common.swaglog import cloudlog |
|
|
|
from openpilot.tools.lib.comma_car_segments import get_url as get_comma_segments_url |
|
|
|
from openpilot.tools.lib.comma_car_segments import get_url as get_comma_segments_url |
|
|
|
from openpilot.tools.lib.openpilotci import get_url |
|
|
|
from openpilot.tools.lib.openpilotci import get_url |
|
|
|
from openpilot.tools.lib.filereader import FileReader, file_exists |
|
|
|
from openpilot.tools.lib.filereader import FileReader, file_exists |
|
|
|
@ -71,8 +72,8 @@ class _LogFileReader: |
|
|
|
class ReadMode(enum.StrEnum): |
|
|
|
class ReadMode(enum.StrEnum): |
|
|
|
RLOG = "r" # only read rlogs |
|
|
|
RLOG = "r" # only read rlogs |
|
|
|
QLOG = "q" # only read qlogs |
|
|
|
QLOG = "q" # only read qlogs |
|
|
|
#AUTO = "a" # default to rlogs, fallback to qlogs, not supported yet |
|
|
|
AUTO = "a" # default to rlogs, fallback to qlogs |
|
|
|
|
|
|
|
AUTO_INTERACIVE = "i" # default to rlogs, fallback to qlogs with a prompt from the user |
|
|
|
|
|
|
|
|
|
|
|
def create_slice_from_string(s: str): |
|
|
|
def create_slice_from_string(s: str): |
|
|
|
m = re.fullmatch(RE.SLICE, s) |
|
|
|
m = re.fullmatch(RE.SLICE, s) |
|
|
|
@ -86,44 +87,86 @@ def create_slice_from_string(s: str): |
|
|
|
return start |
|
|
|
return start |
|
|
|
return slice(start, end, step) |
|
|
|
return slice(start, end, step) |
|
|
|
|
|
|
|
|
|
|
|
def parse_slice(sr: SegmentRange, route: Route): |
|
|
|
def auto_strategy(rlog_paths, qlog_paths, interactive): |
|
|
|
segs = np.arange(route.max_seg_number+1) |
|
|
|
# auto select logs based on availability |
|
|
|
|
|
|
|
if any(rlog is None or not file_exists(rlog) for rlog in rlog_paths): |
|
|
|
|
|
|
|
if interactive: |
|
|
|
|
|
|
|
if input("Some rlogs were not found, would you like to fallback to qlogs for those segments? (y/n) ").lower() != "y": |
|
|
|
|
|
|
|
return rlog_paths |
|
|
|
|
|
|
|
else: |
|
|
|
|
|
|
|
cloudlog.warning("Some rlogs were not found, falling back to qlogs for those segments...") |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return [rlog if (rlog is not None and file_exists(rlog)) else (qlog if (qlog is not None and file_exists(qlog)) else None) |
|
|
|
|
|
|
|
for (rlog, qlog) in zip(rlog_paths, qlog_paths, strict=True)] |
|
|
|
|
|
|
|
return rlog_paths |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def apply_strategy(mode: ReadMode, rlog_paths, qlog_paths): |
|
|
|
|
|
|
|
if mode == ReadMode.RLOG: |
|
|
|
|
|
|
|
return rlog_paths |
|
|
|
|
|
|
|
elif mode == ReadMode.QLOG: |
|
|
|
|
|
|
|
return qlog_paths |
|
|
|
|
|
|
|
elif mode == ReadMode.AUTO: |
|
|
|
|
|
|
|
return auto_strategy(rlog_paths, qlog_paths, False) |
|
|
|
|
|
|
|
elif mode == ReadMode.AUTO_INTERACIVE: |
|
|
|
|
|
|
|
return auto_strategy(rlog_paths, qlog_paths, True) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def parse_slice(sr: SegmentRange): |
|
|
|
s = create_slice_from_string(sr._slice) |
|
|
|
s = create_slice_from_string(sr._slice) |
|
|
|
return segs[s] if isinstance(s, slice) else [segs[s]] |
|
|
|
if isinstance(s, slice): |
|
|
|
|
|
|
|
if s.stop is None or s.stop < 0 or (s.start is not None and s.start < 0): # we need the number of segments in order to parse this slice |
|
|
|
|
|
|
|
segs = np.arange(sr.get_max_seg_number()+1) |
|
|
|
|
|
|
|
else: |
|
|
|
|
|
|
|
segs = np.arange(s.stop + 1) |
|
|
|
|
|
|
|
return segs[s] |
|
|
|
|
|
|
|
else: |
|
|
|
|
|
|
|
if s < 0: |
|
|
|
|
|
|
|
s = sr.get_max_seg_number() + s + 1 |
|
|
|
|
|
|
|
return [s] |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def comma_api_source(sr: SegmentRange, mode: ReadMode): |
|
|
|
|
|
|
|
segs = parse_slice(sr) |
|
|
|
|
|
|
|
|
|
|
|
def comma_api_source(sr: SegmentRange, route: Route, mode=ReadMode.RLOG): |
|
|
|
route = Route(sr.route_name) |
|
|
|
segs = parse_slice(sr, route) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
log_paths = route.log_paths() if mode == ReadMode.RLOG else route.qlog_paths() |
|
|
|
rlog_paths = [route.log_paths()[seg] for seg in segs] |
|
|
|
|
|
|
|
qlog_paths = [route.log_paths()[seg] for seg in segs] |
|
|
|
|
|
|
|
|
|
|
|
invalid_segs = [seg for seg in segs if log_paths[seg] is None] |
|
|
|
return apply_strategy(mode, rlog_paths, qlog_paths) |
|
|
|
|
|
|
|
|
|
|
|
assert not len(invalid_segs), f"Some of the requested segments are not available: {invalid_segs}" |
|
|
|
def internal_source(sr: SegmentRange, mode: ReadMode): |
|
|
|
|
|
|
|
segs = parse_slice(sr) |
|
|
|
|
|
|
|
|
|
|
|
return [(log_paths[seg]) for seg in segs] |
|
|
|
def get_internal_url(sr: SegmentRange, seg, file): |
|
|
|
|
|
|
|
return f"cd:/{sr.dongle_id}/{sr.timestamp}/{seg}/{file}.bz2" |
|
|
|
|
|
|
|
|
|
|
|
def internal_source(sr: SegmentRange, route: Route, mode=ReadMode.RLOG): |
|
|
|
rlog_paths = [get_internal_url(sr, seg, "rlog") for seg in segs] |
|
|
|
segs = parse_slice(sr, route) |
|
|
|
qlog_paths = [get_internal_url(sr, seg, "qlog") for seg in segs] |
|
|
|
|
|
|
|
|
|
|
|
return [f"cd:/{sr.dongle_id}/{sr.timestamp}/{seg}/{'rlog' if mode == ReadMode.RLOG else 'qlog'}.bz2" for seg in segs] |
|
|
|
return apply_strategy(mode, rlog_paths, qlog_paths) |
|
|
|
|
|
|
|
|
|
|
|
def openpilotci_source(sr: SegmentRange, route: Route, mode=ReadMode.RLOG): |
|
|
|
def openpilotci_source(sr: SegmentRange, mode: ReadMode): |
|
|
|
segs = parse_slice(sr, route) |
|
|
|
segs = parse_slice(sr) |
|
|
|
|
|
|
|
|
|
|
|
return [get_url(sr.route_name, seg, 'rlog' if mode == ReadMode.RLOG else 'qlog') for seg in segs] |
|
|
|
rlog_paths = [get_url(sr.route_name, seg, "rlog") for seg in segs] |
|
|
|
|
|
|
|
qlog_paths = [get_url(sr.route_name, seg, "qlog") for seg in segs] |
|
|
|
|
|
|
|
|
|
|
|
def comma_car_segments_source(sr: SegmentRange, route: Route, mode=ReadMode.RLOG): |
|
|
|
return apply_strategy(mode, rlog_paths, qlog_paths) |
|
|
|
segs = parse_slice(sr, route) |
|
|
|
|
|
|
|
|
|
|
|
def comma_car_segments_source(sr: SegmentRange, mode=ReadMode.RLOG): |
|
|
|
|
|
|
|
segs = parse_slice(sr) |
|
|
|
|
|
|
|
|
|
|
|
return [get_comma_segments_url(sr.route_name, seg) for seg in segs] |
|
|
|
return [get_comma_segments_url(sr.route_name, seg) for seg in segs] |
|
|
|
|
|
|
|
|
|
|
|
def direct_source(file_or_url): |
|
|
|
def direct_source(file_or_url): |
|
|
|
return [file_or_url] |
|
|
|
return [file_or_url] |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_invalid_files(files): |
|
|
|
|
|
|
|
return [f for f in files if f is None or not file_exists(f)] |
|
|
|
|
|
|
|
|
|
|
|
def check_source(source, *args): |
|
|
|
def check_source(source, *args): |
|
|
|
try: |
|
|
|
try: |
|
|
|
files = source(*args) |
|
|
|
files = source(*args) |
|
|
|
assert all(file_exists(f) for f in files) |
|
|
|
assert len(get_invalid_files(files)) == 0 |
|
|
|
return True, files |
|
|
|
return True, files |
|
|
|
except Exception: |
|
|
|
except Exception: |
|
|
|
return False, None |
|
|
|
return False, None |
|
|
|
@ -176,11 +219,10 @@ class LogReader: |
|
|
|
return direct_source(identifier) |
|
|
|
return direct_source(identifier) |
|
|
|
|
|
|
|
|
|
|
|
sr = SegmentRange(parsed) |
|
|
|
sr = SegmentRange(parsed) |
|
|
|
route = Route(sr.route_name) |
|
|
|
|
|
|
|
mode = self.default_mode if sr.selector is None else ReadMode(sr.selector) |
|
|
|
mode = self.default_mode if sr.selector is None else ReadMode(sr.selector) |
|
|
|
source = self.default_source if source is None else source |
|
|
|
source = self.default_source if source is None else source |
|
|
|
|
|
|
|
|
|
|
|
return source(sr, route, mode) |
|
|
|
return source(sr, mode) |
|
|
|
|
|
|
|
|
|
|
|
def __init__(self, identifier: str | List[str], default_mode=ReadMode.RLOG, default_source=auto_source, sort_by_time=False, only_union_types=False): |
|
|
|
def __init__(self, identifier: str | List[str], default_mode=ReadMode.RLOG, default_source=auto_source, sort_by_time=False, only_union_types=False): |
|
|
|
self.default_mode = default_mode |
|
|
|
self.default_mode = default_mode |
|
|
|
@ -209,6 +251,9 @@ class LogReader: |
|
|
|
|
|
|
|
|
|
|
|
def reset(self): |
|
|
|
def reset(self): |
|
|
|
self.logreader_identifiers = self._parse_identifiers(self.identifier) |
|
|
|
self.logreader_identifiers = self._parse_identifiers(self.identifier) |
|
|
|
|
|
|
|
invalid_count = len(get_invalid_files(self.logreader_identifiers)) |
|
|
|
|
|
|
|
assert invalid_count == 0, f"{invalid_count}/{len(self.logreader_identifiers)} invalid log(s) found, please ensure all logs \ |
|
|
|
|
|
|
|
are uploaded or auto fallback to qlogs with '/a' selector at the end of the route name." |
|
|
|
|
|
|
|
|
|
|
|
@staticmethod |
|
|
|
@staticmethod |
|
|
|
def from_bytes(dat): |
|
|
|
def from_bytes(dat): |
|
|
|
|
Cameron Clough
2 years ago