diff --git a/selfdrive/test/openpilotci.py b/selfdrive/test/openpilotci.py index 0fc763002b..4e0da1bccf 100755 --- a/selfdrive/test/openpilotci.py +++ b/selfdrive/test/openpilotci.py @@ -55,6 +55,7 @@ def upload_bytes(data: Union[bytes, IO], blob_name: str) -> str: container_name=OPENPILOT_CI_CONTAINER, blob_name=blob_name, credential=get_azure_credential(), + overwrite=False, ) blob.upload_blob(data) return BASE_URL + blob_name diff --git a/selfdrive/test/update_ci_routes.py b/selfdrive/test/update_ci_routes.py index 4ec8b689a5..ccaa7f9b84 100755 --- a/selfdrive/test/update_ci_routes.py +++ b/selfdrive/test/update_ci_routes.py @@ -1,4 +1,6 @@ #!/usr/bin/env python3 +import os +import re import subprocess import sys from functools import lru_cache @@ -10,7 +12,7 @@ from tqdm import tqdm from openpilot.selfdrive.car.tests.routes import routes as test_car_models_routes from openpilot.selfdrive.test.process_replay.test_processes import source_segments as replay_segments from openpilot.selfdrive.test.openpilotci import (DATA_CI_ACCOUNT, DATA_CI_ACCOUNT_URL, OPENPILOT_CI_CONTAINER, - DATA_CI_CONTAINER, get_azure_credential, get_container_sas) + DATA_CI_CONTAINER, get_azure_credential, get_container_sas, upload_file) DATA_PROD_ACCOUNT = "commadata2" DATA_PROD_CONTAINER = "commadata2" @@ -30,23 +32,16 @@ def get_azure_keys(): def upload_route(path: str, exclude_patterns: Optional[Iterable[str]] = None) -> None: - # TODO: use azure-storage-blob instead of azcopy, simplifies auth - dest_key = get_container_sas(DATA_CI_ACCOUNT, OPENPILOT_CI_CONTAINER) if exclude_patterns is None: - exclude_patterns = ['*/dcamera.hevc'] + exclude_patterns = [r'dcamera\.hevc'] r, n = path.rsplit("--", 1) r = '/'.join(r.split('/')[-2:]) # strip out anything extra in the path destpath = f"{r}/{n}" - cmd = [ - "azcopy", - "copy", - f"{path}/*", - f"https://{DATA_CI_ACCOUNT}.blob.core.windows.net/{OPENPILOT_CI_CONTAINER}/{destpath}?{dest_key}", - "--recursive=false", - "--overwrite=false", - ] + [f"--exclude-pattern={p}" for p in exclude_patterns] - subprocess.check_call(cmd) + for file in os.listdir(path): + if any(re.search(pattern, file) for pattern in exclude_patterns): + continue + upload_file(os.path.join(path, file), f"{destpath}/{file}") def sync_to_ci_public(route: str) -> bool: