azure_container: pass the filename explicitely when using get_url (#33407)

* Pass filename explicitely when using azure container

* Do not update segments in test_processes

* openpilotci_source_zst

* Fix static analysisc

selfdrive/test/process_replay/model_replay.py

@@ -73,11 +73,11 @@ if __name__ == "__main__":
   ref_commit_fn = os.path.join(replay_dir, "model_replay_ref_commit")
 
   # load logs
-  lr = list(LogReader(get_url(TEST_ROUTE, SEGMENT)))
+  lr = list(LogReader(get_url(TEST_ROUTE, SEGMENT, "rlog.bz2")))
   frs = {
-    'roadCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, log_type="fcamera"), readahead=True),
+    'roadCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, "fcamera.hevc"), readahead=True),
-    'driverCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, log_type="dcamera"), readahead=True),
+    'driverCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, "dcamera.hevc"), readahead=True),
-    'wideRoadCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, log_type="ecamera"), readahead=True)
+    'wideRoadCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, "ecamera.hevc"), readahead=True)
   }
 
   log_msgs = []

selfdrive/test/process_replay/test_processes.py

@@ -86,7 +86,7 @@ def run_test_process(data):
 
 def get_log_data(segment):
   r, n = segment.rsplit("--", 1)
-  with FileReader(get_url(r, n)) as f:
+  with FileReader(get_url(r, n, "rlog.bz2")) as f:
     return (segment, f.read())
 
 
@@ -199,7 +199,7 @@ if __name__ == "__main__":
 
         cur_log_fn = os.path.join(FAKEDATA, f"{segment}_{cfg.proc_name}_{cur_commit}.zst")
         if args.update_refs:  # reference logs will not exist if routes were just regenerated
-          ref_log_path = get_url(*segment.rsplit("--", 1))
+          ref_log_path = get_url(*segment.rsplit("--", 1,), "rlog.bz2")
         else:
           ref_log_fn = os.path.join(FAKEDATA, f"{segment}_{cfg.proc_name}_{ref_commit}.zst")
           ref_log_path = ref_log_fn if os.path.exists(ref_log_fn) else BASE_URL + os.path.basename(ref_log_fn)

selfdrive/test/process_replay/test_regen.py

@@ -15,13 +15,13 @@ TESTED_SEGMENTS = [
 
 
 def ci_setup_data_readers(route, sidx):
-  lr = LogReader(get_url(route, sidx, "rlog"))
+  lr = LogReader(get_url(route, sidx, "rlog.bz2"))
   frs = {
-    'roadCameraState': FrameReader(get_url(route, sidx, "fcamera")),
+    'roadCameraState': FrameReader(get_url(route, sidx, "fcamera.hevc")),
     'driverCameraState': DummyFrameReader.zero_dcamera()
   }
   if next((True for m in lr if m.which() == "wideRoadCameraState"), False):
-    frs["wideRoadCameraState"] = FrameReader(get_url(route, sidx, "ecamera"))
+    frs["wideRoadCameraState"] = FrameReader(get_url(route, sidx, "ecamera.hevc"))
 
   return lr, frs
 

tools/lib/azure_container.py

@@ -53,9 +53,8 @@ class AzureContainer:
     key = get_container_sas(self.ACCOUNT, self.CONTAINER)
     return client, key
 
-  def get_url(self, route_name: str, segment_num, log_type="rlog") -> str:
+  def get_url(self, route_name: str, segment_num: str, filename: str) -> str:
-    ext = "hevc" if log_type.endswith('camera') else "bz2"
+    return self.BASE_URL + f"{route_name.replace('|', '/')}/{segment_num}/{filename}"
-    return self.BASE_URL + f"{route_name.replace('|', '/')}/{segment_num}/{log_type}.{ext}"
 
   def upload_bytes(self, data: bytes | IO, blob_name: str, overwrite=False) -> str:
     from azure.storage.blob import BlobClient

tools/lib/logreader.py

@@ -163,13 +163,17 @@ def internal_source_zst(sr: SegmentRange, mode: ReadMode, file_ext: str = "zst")
   return internal_source(sr, mode, file_ext)
 
 
-def openpilotci_source(sr: SegmentRange, mode: ReadMode) -> LogPaths:
+def openpilotci_source(sr: SegmentRange, mode: ReadMode, file_ext: str = "bz2") -> LogPaths:
-  rlog_paths = [get_url(sr.route_name, seg, "rlog") for seg in sr.seg_idxs]
+  rlog_paths = [get_url(sr.route_name, seg, f"rlog.{file_ext}") for seg in sr.seg_idxs]
-  qlog_paths = [get_url(sr.route_name, seg, "qlog") for seg in sr.seg_idxs]
+  qlog_paths = [get_url(sr.route_name, seg, f"qlog.{file_ext}") for seg in sr.seg_idxs]
 
   return apply_strategy(mode, rlog_paths, qlog_paths)
 
 
+def openpilotci_source_zst(sr: SegmentRange, mode: ReadMode) -> LogPaths:
+  return openpilotci_source(sr, mode, "zst")
+
+
 def comma_car_segments_source(sr: SegmentRange, mode=ReadMode.RLOG) -> LogPaths:
   return [get_comma_segments_url(sr.route_name, seg) for seg in sr.seg_idxs]
 
@@ -201,7 +205,8 @@ def auto_source(sr: SegmentRange, mode=ReadMode.RLOG) -> LogPaths:
   if mode == ReadMode.SANITIZED:
     return comma_car_segments_source(sr, mode)
 
-  SOURCES: list[Source] = [internal_source, internal_source_zst, openpilotci_source, comma_api_source, comma_car_segments_source, testing_closet_source,]
+  SOURCES: list[Source] = [internal_source, internal_source_zst, openpilotci_source, openpilotci_source_zst,
+                           comma_api_source, comma_car_segments_source, testing_closet_source,]
   exceptions = {}
 
   # for automatic fallback modes, auto_source needs to first check if rlogs exist for any source

tools/replay/unlog_ci_segment.py

@@ -24,8 +24,8 @@ def input_ready():
 def replay(route, segment, loop):
   route = route.replace('|', '/')
 
-  lr = LogReader(get_url(route, segment))
+  lr = LogReader(get_url(route, segment, "rlog.bz2"))
-  fr = FrameReader(get_url(route, segment, "fcamera"), readahead=True)
+  fr = FrameReader(get_url(route, segment, "fcamera.hevc"), readahead=True)
 
   # Build mapping from frameId to segmentId from roadEncodeIdx, type == fullHEVC
   msgs = [m for m in lr if m.which() not in IGNORE]