diff --git a/selfdrive/test/process_replay/model_replay.py b/selfdrive/test/process_replay/model_replay.py index a58a58107e..4402ca31d6 100755 --- a/selfdrive/test/process_replay/model_replay.py +++ b/selfdrive/test/process_replay/model_replay.py @@ -73,11 +73,11 @@ if __name__ == "__main__": ref_commit_fn = os.path.join(replay_dir, "model_replay_ref_commit") # load logs - lr = list(LogReader(get_url(TEST_ROUTE, SEGMENT))) + lr = list(LogReader(get_url(TEST_ROUTE, SEGMENT, "rlog.bz2"))) frs = { - 'roadCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, log_type="fcamera"), readahead=True), - 'driverCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, log_type="dcamera"), readahead=True), - 'wideRoadCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, log_type="ecamera"), readahead=True) + 'roadCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, "fcamera.hevc"), readahead=True), + 'driverCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, "dcamera.hevc"), readahead=True), + 'wideRoadCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, "ecamera.hevc"), readahead=True) } log_msgs = [] diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index fbd23adf89..faad05c0f2 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -86,7 +86,7 @@ def run_test_process(data): def get_log_data(segment): r, n = segment.rsplit("--", 1) - with FileReader(get_url(r, n)) as f: + with FileReader(get_url(r, n, "rlog.bz2")) as f: return (segment, f.read()) @@ -199,7 +199,7 @@ if __name__ == "__main__": cur_log_fn = os.path.join(FAKEDATA, f"{segment}_{cfg.proc_name}_{cur_commit}.zst") if args.update_refs: # reference logs will not exist if routes were just regenerated - ref_log_path = get_url(*segment.rsplit("--", 1)) + ref_log_path = get_url(*segment.rsplit("--", 1,), "rlog.bz2") else: ref_log_fn = os.path.join(FAKEDATA, f"{segment}_{cfg.proc_name}_{ref_commit}.zst") ref_log_path = ref_log_fn if os.path.exists(ref_log_fn) else BASE_URL + os.path.basename(ref_log_fn) diff --git a/selfdrive/test/process_replay/test_regen.py b/selfdrive/test/process_replay/test_regen.py index 17fefcb497..6c4b48c8d5 100644 --- a/selfdrive/test/process_replay/test_regen.py +++ b/selfdrive/test/process_replay/test_regen.py @@ -15,13 +15,13 @@ TESTED_SEGMENTS = [ def ci_setup_data_readers(route, sidx): - lr = LogReader(get_url(route, sidx, "rlog")) + lr = LogReader(get_url(route, sidx, "rlog.bz2")) frs = { - 'roadCameraState': FrameReader(get_url(route, sidx, "fcamera")), + 'roadCameraState': FrameReader(get_url(route, sidx, "fcamera.hevc")), 'driverCameraState': DummyFrameReader.zero_dcamera() } if next((True for m in lr if m.which() == "wideRoadCameraState"), False): - frs["wideRoadCameraState"] = FrameReader(get_url(route, sidx, "ecamera")) + frs["wideRoadCameraState"] = FrameReader(get_url(route, sidx, "ecamera.hevc")) return lr, frs diff --git a/tools/lib/azure_container.py b/tools/lib/azure_container.py index a5d650e98f..7ba2e1515f 100644 --- a/tools/lib/azure_container.py +++ b/tools/lib/azure_container.py @@ -53,9 +53,8 @@ class AzureContainer: key = get_container_sas(self.ACCOUNT, self.CONTAINER) return client, key - def get_url(self, route_name: str, segment_num, log_type="rlog") -> str: - ext = "hevc" if log_type.endswith('camera') else "bz2" - return self.BASE_URL + f"{route_name.replace('|', '/')}/{segment_num}/{log_type}.{ext}" + def get_url(self, route_name: str, segment_num: str, filename: str) -> str: + return self.BASE_URL + f"{route_name.replace('|', '/')}/{segment_num}/{filename}" def upload_bytes(self, data: bytes | IO, blob_name: str, overwrite=False) -> str: from azure.storage.blob import BlobClient diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index 815bd4b796..c12b14ebd4 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -163,13 +163,17 @@ def internal_source_zst(sr: SegmentRange, mode: ReadMode, file_ext: str = "zst") return internal_source(sr, mode, file_ext) -def openpilotci_source(sr: SegmentRange, mode: ReadMode) -> LogPaths: - rlog_paths = [get_url(sr.route_name, seg, "rlog") for seg in sr.seg_idxs] - qlog_paths = [get_url(sr.route_name, seg, "qlog") for seg in sr.seg_idxs] +def openpilotci_source(sr: SegmentRange, mode: ReadMode, file_ext: str = "bz2") -> LogPaths: + rlog_paths = [get_url(sr.route_name, seg, f"rlog.{file_ext}") for seg in sr.seg_idxs] + qlog_paths = [get_url(sr.route_name, seg, f"qlog.{file_ext}") for seg in sr.seg_idxs] return apply_strategy(mode, rlog_paths, qlog_paths) +def openpilotci_source_zst(sr: SegmentRange, mode: ReadMode) -> LogPaths: + return openpilotci_source(sr, mode, "zst") + + def comma_car_segments_source(sr: SegmentRange, mode=ReadMode.RLOG) -> LogPaths: return [get_comma_segments_url(sr.route_name, seg) for seg in sr.seg_idxs] @@ -201,7 +205,8 @@ def auto_source(sr: SegmentRange, mode=ReadMode.RLOG) -> LogPaths: if mode == ReadMode.SANITIZED: return comma_car_segments_source(sr, mode) - SOURCES: list[Source] = [internal_source, internal_source_zst, openpilotci_source, comma_api_source, comma_car_segments_source, testing_closet_source,] + SOURCES: list[Source] = [internal_source, internal_source_zst, openpilotci_source, openpilotci_source_zst, + comma_api_source, comma_car_segments_source, testing_closet_source,] exceptions = {} # for automatic fallback modes, auto_source needs to first check if rlogs exist for any source diff --git a/tools/replay/unlog_ci_segment.py b/tools/replay/unlog_ci_segment.py index a2011f6876..adc0b19e9b 100755 --- a/tools/replay/unlog_ci_segment.py +++ b/tools/replay/unlog_ci_segment.py @@ -24,8 +24,8 @@ def input_ready(): def replay(route, segment, loop): route = route.replace('|', '/') - lr = LogReader(get_url(route, segment)) - fr = FrameReader(get_url(route, segment, "fcamera"), readahead=True) + lr = LogReader(get_url(route, segment, "rlog.bz2")) + fr = FrameReader(get_url(route, segment, "fcamera.hevc"), readahead=True) # Build mapping from frameId to segmentId from roadEncodeIdx, type == fullHEVC msgs = [m for m in lr if m.which() not in IGNORE]