You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
74 lines
2.3 KiB
74 lines
2.3 KiB
#!/usr/bin/env python3
|
|
import tempfile
|
|
import shutil
|
|
import subprocess
|
|
from common.basedir import BASEDIR
|
|
from azure.storage.blob import BlockBlobService
|
|
|
|
from selfdrive.test.test_car_models import routes as test_car_models_routes
|
|
from selfdrive.test.process_replay.test_processes import segments as replay_segments
|
|
from xx.chffr.lib import azureutil
|
|
from xx.chffr.lib.storage import upload_dir_serial, download_dir_tpe
|
|
from xx.chffr.lib.storage import _DATA_ACCOUNT_PRODUCTION, _DATA_ACCOUNT_CI, _DATA_BUCKET_PRODUCTION, _DATA_BUCKET_CI
|
|
|
|
SOURCES = [
|
|
(_DATA_ACCOUNT_PRODUCTION, _DATA_BUCKET_PRODUCTION),
|
|
(_DATA_ACCOUNT_PRODUCTION, "preserve"),
|
|
(_DATA_ACCOUNT_CI, "commadataci"),
|
|
]
|
|
|
|
DEST_KEY = azureutil.get_user_token(_DATA_ACCOUNT_CI, "openpilotci")
|
|
SOURCE_KEYS = [azureutil.get_user_token(account, bucket) for account, bucket in SOURCES]
|
|
SERVICE = BlockBlobService(_DATA_ACCOUNT_CI, sas_token=DEST_KEY)
|
|
|
|
|
|
def sync_to_ci_public(route):
|
|
print(f"Uploading {route}")
|
|
key_prefix = route.replace('|', '/')
|
|
dongle_id = key_prefix.split('/')[0]
|
|
|
|
if next(azureutil.list_all_blobs(SERVICE, "openpilotci", prefix=key_prefix), None) is not None:
|
|
print("Already synced")
|
|
return True
|
|
|
|
for (source_account, source_bucket), source_key in zip(SOURCES, SOURCE_KEYS):
|
|
print(f"Trying {source_account}/{source_bucket}")
|
|
cmd = [
|
|
f"{BASEDIR}/external/bin/azcopy",
|
|
"copy",
|
|
"https://{}.blob.core.windows.net/{}/{}/?{}".format(source_account, source_bucket, key_prefix, source_key),
|
|
"https://{}.blob.core.windows.net/{}/{}/?{}".format(_DATA_ACCOUNT_CI, "openpilotci", dongle_id, DEST_KEY),
|
|
"--recursive=true",
|
|
"--overwrite=false",
|
|
"--exclude=*/dcamera.hevc",
|
|
]
|
|
|
|
try:
|
|
result = subprocess.call(cmd, stdout=subprocess.DEVNULL)
|
|
if result == 0:
|
|
print("Success")
|
|
return True
|
|
except subprocess.CalledProcessError:
|
|
print("Failed")
|
|
|
|
return False
|
|
|
|
|
|
if __name__ == "__main__":
|
|
failed_routes = []
|
|
|
|
# sync process replay routes
|
|
for s in replay_segments:
|
|
route_name, _ = s[1].rsplit('--', 1)
|
|
if not sync_to_ci_public(route_name):
|
|
failed_routes.append(route_name)
|
|
|
|
# sync test_car_models routes
|
|
for r in list(test_car_models_routes.keys()):
|
|
if not sync_to_ci_public(r):
|
|
failed_routes.append(r)
|
|
|
|
|
|
if len(failed_routes):
|
|
print("failed routes:")
|
|
print(failed_routes)
|
|
|