|
|
|
@ -9,8 +9,8 @@ from tqdm import tqdm |
|
|
|
|
|
|
|
|
|
from openpilot.selfdrive.car.tests.routes import routes as test_car_models_routes |
|
|
|
|
from openpilot.selfdrive.test.process_replay.test_processes import source_segments as replay_segments |
|
|
|
|
from openpilot.selfdrive.test.openpilotci import (DATA_CI_ACCOUNT, DATA_CI_ACCOUNT_URL, DATA_CI_CONTAINER, |
|
|
|
|
get_azure_credential, get_container_sas) |
|
|
|
|
from openpilot.selfdrive.test.openpilotci import (DATA_CI_ACCOUNT, DATA_CI_ACCOUNT_URL, OPENPILOT_CI_CONTAINER, |
|
|
|
|
DATA_CI_CONTAINER, get_azure_credential, get_container_sas) |
|
|
|
|
|
|
|
|
|
DATA_PROD_ACCOUNT = "commadata2" |
|
|
|
|
DATA_PROD_CONTAINER = "commadata2" |
|
|
|
@ -23,15 +23,15 @@ SOURCES = [ |
|
|
|
|
|
|
|
|
|
@lru_cache |
|
|
|
|
def get_azure_keys(): |
|
|
|
|
dest_container = ContainerClient(DATA_CI_ACCOUNT_URL, DATA_CI_CONTAINER, credential=get_azure_credential()) |
|
|
|
|
dest_key = get_container_sas(DATA_CI_ACCOUNT, DATA_CI_CONTAINER) |
|
|
|
|
dest_container = ContainerClient(DATA_CI_ACCOUNT_URL, OPENPILOT_CI_CONTAINER, credential=get_azure_credential()) |
|
|
|
|
dest_key = get_container_sas(DATA_CI_ACCOUNT, OPENPILOT_CI_CONTAINER) |
|
|
|
|
source_keys = [get_container_sas(*s) for s in SOURCES] |
|
|
|
|
return dest_container, dest_key, source_keys |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def upload_route(path: str, exclude_patterns: Optional[Iterable[str]] = None) -> None: |
|
|
|
|
# TODO: use azure-storage-blob instead of azcopy, simplifies auth |
|
|
|
|
dest_key = get_container_sas(DATA_CI_ACCOUNT, DATA_CI_CONTAINER) |
|
|
|
|
dest_key = get_container_sas(DATA_CI_ACCOUNT, OPENPILOT_CI_CONTAINER) |
|
|
|
|
if exclude_patterns is None: |
|
|
|
|
exclude_patterns = ['*/dcamera.hevc'] |
|
|
|
|
|
|
|
|
@ -42,7 +42,7 @@ def upload_route(path: str, exclude_patterns: Optional[Iterable[str]] = None) -> |
|
|
|
|
"azcopy", |
|
|
|
|
"copy", |
|
|
|
|
f"{path}/*", |
|
|
|
|
f"https://{DATA_CI_ACCOUNT}.blob.core.windows.net/{DATA_CI_CONTAINER}/{destpath}?{dest_key}", |
|
|
|
|
f"https://{DATA_CI_ACCOUNT}.blob.core.windows.net/{OPENPILOT_CI_CONTAINER}/{destpath}?{dest_key}", |
|
|
|
|
"--recursive=false", |
|
|
|
|
"--overwrite=false", |
|
|
|
|
] + [f"--exclude-pattern={p}" for p in exclude_patterns] |
|
|
|
@ -65,7 +65,7 @@ def sync_to_ci_public(route: str) -> bool: |
|
|
|
|
"azcopy", |
|
|
|
|
"copy", |
|
|
|
|
f"https://{source_account}.blob.core.windows.net/{source_bucket}/{key_prefix}?{source_key}", |
|
|
|
|
f"https://{DATA_CI_ACCOUNT}.blob.core.windows.net/{DATA_CI_CONTAINER}/{dongle_id}?{dest_key}", |
|
|
|
|
f"https://{DATA_CI_ACCOUNT}.blob.core.windows.net/{OPENPILOT_CI_CONTAINER}/{dongle_id}?{dest_key}", |
|
|
|
|
"--recursive=true", |
|
|
|
|
"--overwrite=false", |
|
|
|
|
"--exclude-pattern=*/dcamera.hevc", |
|
|
|
|