diff --git a/selfdrive/test/process_replay/regen.py b/selfdrive/test/process_replay/regen.py index 2653a0126a..c9f9c6c362 100755 --- a/selfdrive/test/process_replay/regen.py +++ b/selfdrive/test/process_replay/regen.py @@ -224,13 +224,14 @@ def migrate_sensorEvents(lr, old_logtime=False): m_dat.sensor = evt.sensor m_dat.type = evt.type m_dat.source = evt.source + if old_logtime: + m_dat.timestamp = evt.timestamp setattr(m_dat, evt.which(), getattr(evt, evt.which())) all_msgs.append(m.as_reader()) return all_msgs - def regen_segment(lr, frs=None, outdir=FAKEDATA, disable_tqdm=False): lr = migrate_carparams(list(lr)) lr = migrate_sensorEvents(list(lr)) diff --git a/selfdrive/test/update_ci_routes.py b/selfdrive/test/update_ci_routes.py index a1e8c35f6b..201ffb745a 100755 --- a/selfdrive/test/update_ci_routes.py +++ b/selfdrive/test/update_ci_routes.py @@ -14,11 +14,16 @@ SOURCES = [ (_DATA_ACCOUNT_CI, "commadataci"), ] -DEST_KEY = azureutil.get_user_token(_DATA_ACCOUNT_CI, "openpilotci") -SOURCE_KEYS = [azureutil.get_user_token(account, bucket) for account, bucket in SOURCES] -SERVICE = BlockBlobService(_DATA_ACCOUNT_CI, sas_token=DEST_KEY) + +def get_azure_keys(): + dest_key = azureutil.get_user_token(_DATA_ACCOUNT_CI, "openpilotci") + source_keys = [azureutil.get_user_token(account, bucket) for account, bucket in SOURCES] + service = BlockBlobService(_DATA_ACCOUNT_CI, sas_token=dest_key) + return dest_key, source_keys, service + def upload_route(path, exclude_patterns=None): + dest_key, _, _ = get_azure_keys() if exclude_patterns is None: exclude_patterns = ['*/dcamera.hevc'] @@ -29,27 +34,28 @@ def upload_route(path, exclude_patterns=None): "azcopy", "copy", f"{path}/*", - f"https://{_DATA_ACCOUNT_CI}.blob.core.windows.net/openpilotci/{destpath}?{DEST_KEY}", + f"https://{_DATA_ACCOUNT_CI}.blob.core.windows.net/openpilotci/{destpath}?{dest_key}", "--recursive=false", "--overwrite=false", ] + [f"--exclude-pattern={p}" for p in exclude_patterns] subprocess.check_call(cmd) def sync_to_ci_public(route): + dest_key, source_keys, service = get_azure_keys() key_prefix = route.replace('|', '/') dongle_id = key_prefix.split('/')[0] - if next(azureutil.list_all_blobs(SERVICE, "openpilotci", prefix=key_prefix), None) is not None: + if next(azureutil.list_all_blobs(service, "openpilotci", prefix=key_prefix), None) is not None: return True print(f"Uploading {route}") - for (source_account, source_bucket), source_key in zip(SOURCES, SOURCE_KEYS): + for (source_account, source_bucket), source_key in zip(SOURCES, source_keys): print(f"Trying {source_account}/{source_bucket}") cmd = [ "azcopy", "copy", f"https://{source_account}.blob.core.windows.net/{source_bucket}/{key_prefix}?{source_key}", - f"https://{_DATA_ACCOUNT_CI}.blob.core.windows.net/openpilotci/{dongle_id}?{DEST_KEY}", + f"https://{_DATA_ACCOUNT_CI}.blob.core.windows.net/openpilotci/{dongle_id}?{dest_key}", "--recursive=true", "--overwrite=false", "--exclude-pattern=*/dcamera.hevc",