LogReader: try zst on internal source (#32751)

* internal source list files like azure api

* messy but works

* no limit

* simpler

* clean up

* clean up

* clean up

* that's obvious

* better

* we need to unfortunately return a url, so best to take a naive approach for now

* todo

* fix

* clean up
old-commit-hash: b45caf4033
testing-closet^2
Shane Smiskol 1 year ago committed by GitHub
parent 644156420b
commit 9f50db15c0
  1. 10
      tools/lib/logreader.py

@ -130,12 +130,12 @@ def comma_api_source(sr: SegmentRange, mode: ReadMode) -> LogPaths:
return apply_strategy(mode, rlog_paths, qlog_paths, valid_file=valid_file)
def internal_source(sr: SegmentRange, mode: ReadMode) -> LogPaths:
def internal_source(sr: SegmentRange, mode: ReadMode, file_ext: str = "bz2") -> LogPaths:
if not internal_source_available():
raise InternalUnavailableException
def get_internal_url(sr: SegmentRange, seg, file):
return f"cd:/{sr.dongle_id}/{sr.log_id}/{seg}/{file}.bz2"
return f"cd:/{sr.dongle_id}/{sr.log_id}/{seg}/{file}.{file_ext}"
rlog_paths = [get_internal_url(sr, seg, "rlog") for seg in sr.seg_idxs]
qlog_paths = [get_internal_url(sr, seg, "qlog") for seg in sr.seg_idxs]
@ -143,6 +143,10 @@ def internal_source(sr: SegmentRange, mode: ReadMode) -> LogPaths:
return apply_strategy(mode, rlog_paths, qlog_paths)
def internal_source_zst(sr: SegmentRange, mode: ReadMode, file_ext: str = "zst") -> LogPaths:
return internal_source(sr, mode, file_ext)
def openpilotci_source(sr: SegmentRange, mode: ReadMode) -> LogPaths:
rlog_paths = [get_url(sr.route_name, seg, "rlog") for seg in sr.seg_idxs]
qlog_paths = [get_url(sr.route_name, seg, "qlog") for seg in sr.seg_idxs]
@ -175,7 +179,7 @@ def auto_source(sr: SegmentRange, mode=ReadMode.RLOG) -> LogPaths:
if mode == ReadMode.SANITIZED:
return comma_car_segments_source(sr, mode)
SOURCES: list[Source] = [internal_source, openpilotci_source, comma_api_source, comma_car_segments_source,]
SOURCES: list[Source] = [internal_source, internal_source_zst, openpilotci_source, comma_api_source, comma_car_segments_source,]
exceptions = {}
# for automatic fallback modes, auto_source needs to first check if rlogs exist for any source

Loading…
Cancel
Save