CI: use azure-storage-blob for uploading (#30478)

* use azure-storage-blob

* fix pattern

* nsp
old-commit-hash: c2af245b85

selfdrive/test/openpilotci.py

@@ -55,6 +55,7 @@ def upload_bytes(data: Union[bytes, IO], blob_name: str) -> str:
     container_name=OPENPILOT_CI_CONTAINER,
     blob_name=blob_name,
     credential=get_azure_credential(),
+    overwrite=False,
   )
   blob.upload_blob(data)
   return BASE_URL + blob_name

selfdrive/test/update_ci_routes.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+import os
+import re
 import subprocess
 import sys
 from functools import lru_cache
@@ -10,7 +12,7 @@ from tqdm import tqdm
 from openpilot.selfdrive.car.tests.routes import routes as test_car_models_routes
 from openpilot.selfdrive.test.process_replay.test_processes import source_segments as replay_segments
 from openpilot.selfdrive.test.openpilotci import (DATA_CI_ACCOUNT, DATA_CI_ACCOUNT_URL, OPENPILOT_CI_CONTAINER,
-                                                  DATA_CI_CONTAINER, get_azure_credential, get_container_sas)
+                                                  DATA_CI_CONTAINER, get_azure_credential, get_container_sas, upload_file)
 
 DATA_PROD_ACCOUNT = "commadata2"
 DATA_PROD_CONTAINER = "commadata2"
@@ -30,23 +32,16 @@ def get_azure_keys():
 
 
 def upload_route(path: str, exclude_patterns: Optional[Iterable[str]] = None) -> None:
-  # TODO: use azure-storage-blob instead of azcopy, simplifies auth
-  dest_key = get_container_sas(DATA_CI_ACCOUNT, OPENPILOT_CI_CONTAINER)
   if exclude_patterns is None:
-    exclude_patterns = ['*/dcamera.hevc']
+    exclude_patterns = [r'dcamera\.hevc']
 
   r, n = path.rsplit("--", 1)
   r = '/'.join(r.split('/')[-2:])  # strip out anything extra in the path
   destpath = f"{r}/{n}"
-  cmd = [
-    "azcopy",
-    "copy",
-    f"{path}/*",
-    f"https://{DATA_CI_ACCOUNT}.blob.core.windows.net/{OPENPILOT_CI_CONTAINER}/{destpath}?{dest_key}",
-    "--recursive=false",
-    "--overwrite=false",
-  ] + [f"--exclude-pattern={p}" for p in exclude_patterns]
-  subprocess.check_call(cmd)
+  for file in os.listdir(path):
+    if any(re.search(pattern, file) for pattern in exclude_patterns):
+      continue
+    upload_file(os.path.join(path, file), f"{destpath}/{file}")
 
 
 def sync_to_ci_public(route: str) -> bool: