You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
50 lines
1.9 KiB
50 lines
1.9 KiB
5 years ago
|
#!/usr/bin/env python3
|
||
|
|
import os
|
||
|
|
import subprocess
|
||
|
|
import tempfile
|
||
|
|
import shutil
|
||
|
|
from azure.storage.blob import BlockBlobService
|
||
|
|
|
||
|
|
from selfdrive.test.test_car_models import routes as test_car_models_routes, non_public_routes
|
||
|
|
from selfdrive.test.process_replay.test_processes import segments as replay_segments
|
||
|
|
from xx.chffr.lib import azureutil
|
||
|
|
from xx.chffr.lib.storage import upload_dir_serial, download_dir_tpe, key_prefix_exists
|
||
|
|
from xx.chffr.lib.storage import _DATA_ACCOUNT_PRODUCTION, _DATA_ACCOUNT_CI, _DATA_BUCKET_PRODUCTION, _DATA_BUCKET_CI
|
||
|
|
|
||
|
|
sas_token = os.getenv("TOKEN", None)
|
||
|
|
if sas_token is None:
|
||
|
|
sas_token = subprocess.check_output("az storage container generate-sas --account-name commadataci --name openpilotci --https-only --permissions lrw --expiry $(date -u '+%Y-%m-%dT%H:%M:%SZ' -d '+1 hour') --auth-mode login --as-user --output tsv", shell=True).decode().strip("\n")
|
||
|
|
service = BlockBlobService(account_name=_DATA_ACCOUNT_CI, sas_token=sas_token)
|
||
|
|
|
||
|
|
def sync_to_ci_public(service, route):
|
||
|
|
key_prefix = route.replace('|', '/')
|
||
|
|
|
||
|
|
if next(azureutil.list_all_blobs(service, "openpilotci", prefix=key_prefix), None) is not None:
|
||
|
|
return
|
||
|
|
|
||
|
|
print("uploading", route)
|
||
|
|
|
||
|
|
tmpdir = tempfile.mkdtemp()
|
||
|
|
try:
|
||
|
|
print(f"download_dir_tpe({_DATA_ACCOUNT_PRODUCTION}, {_DATA_BUCKET_PRODUCTION}, {key_prefix}, {tmpdir})")
|
||
|
|
|
||
|
|
# production -> openpilotci
|
||
|
|
download_dir_tpe(_DATA_ACCOUNT_PRODUCTION, _DATA_BUCKET_PRODUCTION, tmpdir, key_prefix)
|
||
|
|
|
||
|
|
# commadataci -> openpilotci
|
||
|
|
#download_dir_tpe(_DATA_ACCOUNT_CI, _DATA_BUCKET_CI, tmpdir, key_prefix)
|
||
|
|
|
||
|
|
upload_dir_serial(_DATA_ACCOUNT_CI, "openpilotci", tmpdir, key_prefix)
|
||
|
|
finally:
|
||
|
|
shutil.rmtree(tmpdir)
|
||
|
|
|
||
|
|
# sync process replay routes
|
||
|
|
for s in replay_segments:
|
||
|
|
route_name, _ = s.rsplit('--', 1)
|
||
|
|
sync_to_ci_public(service, route_name)
|
||
|
|
|
||
|
|
# sync test_car_models routes
|
||
|
|
for r in test_car_models_routes:
|
||
|
|
if r not in non_public_routes:
|
||
|
|
sync_to_ci_public(service, r)
|