From 338b7c915fd37345fe925088c7444ddfdbab9e3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20R=C4=85czy?= Date: Mon, 3 Mar 2025 22:10:48 -0500 Subject: [PATCH] regen: use ci bucket for files (#34766) * Remove route meta mode * use get_url * Remove useless import --- selfdrive/test/process_replay/regen.py | 52 +++++++--------------- selfdrive/test/process_replay/regen_all.py | 2 +- 2 files changed, 18 insertions(+), 36 deletions(-) diff --git a/selfdrive/test/process_replay/regen.py b/selfdrive/test/process_replay/regen.py index e87b8347e1..4d00613f0a 100755 --- a/selfdrive/test/process_replay/regen.py +++ b/selfdrive/test/process_replay/regen.py @@ -12,10 +12,9 @@ from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, FAKE check_openpilot_enabled, check_most_messages_valid, get_custom_params_from_lr from openpilot.selfdrive.test.process_replay.vision_meta import DRIVER_CAMERA_FRAME_SIZES from openpilot.selfdrive.test.update_ci_routes import upload_route -from openpilot.tools.lib.route import Route from openpilot.tools.lib.framereader import FrameReader, BaseFrameReader, FrameType from openpilot.tools.lib.logreader import LogReader, LogIterable, save_log - +from openpilot.tools.lib.openpilotci import get_url class DummyFrameReader(BaseFrameReader): def __init__(self, w: int, h: int, frame_count: int, pix_val: int): @@ -55,45 +54,28 @@ def regen_segment( def setup_data_readers( - route: str, sidx: int, use_route_meta: bool, - needs_driver_cam: bool = True, needs_road_cam: bool = True, dummy_driver_cam: bool = False + route: str, sidx: int, needs_driver_cam: bool = True, needs_road_cam: bool = True, dummy_driver_cam: bool = False ) -> tuple[LogReader, dict[str, Any]]: - if use_route_meta: - r = Route(route) - lr = LogReader(r.log_paths()[sidx]) - frs = {} - if needs_road_cam and len(r.camera_paths()) > sidx and r.camera_paths()[sidx] is not None: - frs['roadCameraState'] = FrameReader(r.camera_paths()[sidx]) - if needs_road_cam and len(r.ecamera_paths()) > sidx and r.ecamera_paths()[sidx] is not None: - frs['wideRoadCameraState'] = FrameReader(r.ecamera_paths()[sidx]) - if needs_driver_cam: - if dummy_driver_cam: - frs['driverCameraState'] = DummyFrameReader.zero_dcamera() - elif len(r.dcamera_paths()) > sidx and r.dcamera_paths()[sidx] is not None: - device_type = next(str(msg.initData.deviceType) for msg in lr if msg.which() == "initData") - assert device_type != "neo", "Driver camera not supported on neo segments. Use dummy dcamera." - frs['driverCameraState'] = FrameReader(r.dcamera_paths()[sidx]) - else: - lr = LogReader(f"{route}/{sidx}/r") - frs = {} - if needs_road_cam: - frs['roadCameraState'] = FrameReader(f"cd:/{route.replace('|', '/')}/{sidx}/fcamera.hevc") - if next((True for m in lr if m.which() == "wideRoadCameraState"), False): - frs['wideRoadCameraState'] = FrameReader(f"cd:/{route.replace('|', '/')}/{sidx}/ecamera.hevc") - if needs_driver_cam: - if dummy_driver_cam: - frs['driverCameraState'] = DummyFrameReader.zero_dcamera() - else: - device_type = next(str(msg.initData.deviceType) for msg in lr if msg.which() == "initData") - assert device_type != "neo", "Driver camera not supported on neo segments. Use dummy dcamera." - frs['driverCameraState'] = FrameReader(f"cd:/{route.replace('|', '/')}/{sidx}/dcamera.hevc") + lr = LogReader(f"{route}/{sidx}/r") + frs = {} + if needs_road_cam: + frs['roadCameraState'] = FrameReader(get_url(route, str(sidx), "fcamera.hevc")) + if next((True for m in lr if m.which() == "wideRoadCameraState"), False): + frs['wideRoadCameraState'] = FrameReader(get_url(route, str(sidx), "ecamera.hevc")) + if needs_driver_cam: + if dummy_driver_cam: + frs['driverCameraState'] = DummyFrameReader.zero_dcamera() + else: + device_type = next(str(msg.initData.deviceType) for msg in lr if msg.which() == "initData") + assert device_type != "neo", "Driver camera not supported on neo segments. Use dummy dcamera." + frs['driverCameraState'] = FrameReader(get_url(route, str(sidx), "dcamera.hevc")) return lr, frs def regen_and_save( route: str, sidx: int, processes: str | Iterable[str] = "all", outdir: str = FAKEDATA, - upload: bool = False, use_route_meta: bool = False, disable_tqdm: bool = False, dummy_driver_cam: bool = False + upload: bool = False, disable_tqdm: bool = False, dummy_driver_cam: bool = False ) -> str: if not isinstance(processes, str) and not hasattr(processes, "__iter__"): raise ValueError("whitelist_proc must be a string or iterable") @@ -110,7 +92,7 @@ def regen_and_save( replayed_processes = CONFIGS all_vision_pubs = {pub for cfg in replayed_processes for pub in cfg.vision_pubs} - lr, frs = setup_data_readers(route, sidx, use_route_meta, + lr, frs = setup_data_readers(route, sidx, needs_driver_cam="driverCameraState" in all_vision_pubs, needs_road_cam="roadCameraState" in all_vision_pubs or "wideRoadCameraState" in all_vision_pubs, dummy_driver_cam=dummy_driver_cam) diff --git a/selfdrive/test/process_replay/regen_all.py b/selfdrive/test/process_replay/regen_all.py index 656a5b89e1..78a90b420c 100755 --- a/selfdrive/test/process_replay/regen_all.py +++ b/selfdrive/test/process_replay/regen_all.py @@ -17,7 +17,7 @@ def regen_job(segment, upload, disable_tqdm): sn = SegmentName(segment[1]) fake_dongle_id = 'regen' + ''.join(random.choice('0123456789ABCDEF') for _ in range(11)) try: - relr = regen_and_save(sn.route_name.canonical_name, sn.segment_num, upload=upload, use_route_meta=False, + relr = regen_and_save(sn.route_name.canonical_name, sn.segment_num, upload=upload, outdir=os.path.join(FAKEDATA, fake_dongle_id), disable_tqdm=disable_tqdm, dummy_driver_cam=True) relr = '|'.join(relr.split('/')[-2:]) return f' ("{segment[0]}", "{relr}"), '