jenkins: publish casync releases from device (#32142)

* publish in ci

* overwrite

* publish in ci

* fix

* test it

* Revert "test it"

This reverts commit b3de51dc69.

* use right token

* cleanup after uploading

---------

Co-authored-by: Comma Device <device@comma.ai>

Jenkinsfile

@@ -153,7 +153,7 @@ def build_release(String channel_name) {
       deviceStage("build casync", "tici-needs-can", [], [
         ["build ${channel_name}", "RELEASE=1 OPENPILOT_CHANNEL=${channel_name} BUILD_DIR=/data/openpilot CASYNC_DIR=/data/casync $SOURCE_DIR/release/create_casync_build.sh"],
         ["create manifest", "$SOURCE_DIR/release/create_release_manifest.py /data/manifest.json && cat /data/manifest.json"],
-        //["upload ${channel_name}", "OPENPILOT_CHANNEL=${channel_name} $SOURCE_DIR/release/upload_casync_release.sh"],
+        ["upload and cleanup ${channel_name}", "$SOURCE_DIR/release/upload_casync_release.py /data/casync && rm -rf /data/casync"],
       ])
     }
   )

release/upload_casync_release.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import pathlib
+from openpilot.tools.lib.azure_container import AzureContainer
+
+
+if __name__ == "__main__":
+  del os.environ["AZURE_TOKEN"] # regerenate token for this bucket
+
+  OPENPILOT_RELEASES_CONTAINER = AzureContainer("commadist", "openpilot-releases")
+
+  parser = argparse.ArgumentParser(description='upload casync folder to azure')
+  parser.add_argument("casync_dir", type=str, help="casync directory")
+  args = parser.parse_args()
+
+  casync_dir = pathlib.Path(args.casync_dir)
+
+  for f in casync_dir.rglob("*"):
+    if f.is_file():
+      blob_name = f.relative_to(casync_dir)
+      print(f"uploading {f} to {blob_name}")
+      OPENPILOT_RELEASES_CONTAINER.upload_file(str(f), str(blob_name), overwrite=True)

release/upload_casync_release.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-CASYNC_DIR="${CASYNC_DIR:=/tmp/casync}"
-
-OPENPILOT_RELEASES="https://commadist.blob.core.windows.net/openpilot-releases/"
-
-SAS="$(python -c 'from tools.lib.azure_container import get_container_sas;print(get_container_sas("commadist","openpilot-releases"))')"
-
-azcopy cp "$CASYNC_DIR*" "$OPENPILOT_RELEASES?$SAS" --recursive

tools/lib/azure_container.py

@@ -57,18 +57,18 @@ class AzureContainer:
     ext = "hevc" if log_type.endswith('camera') else "bz2"
     return self.BASE_URL + f"{route_name.replace('|', '/')}/{segment_num}/{log_type}.{ext}"
 
-  def upload_bytes(self, data: bytes | IO, blob_name: str) -> str:
+  def upload_bytes(self, data: bytes | IO, blob_name: str, overwrite=False) -> str:
     from azure.storage.blob import BlobClient
     blob = BlobClient(
       account_url=self.ACCOUNT_URL,
       container_name=self.CONTAINER,
       blob_name=blob_name,
       credential=get_azure_credential(),
-      overwrite=False,
+      overwrite=overwrite,
     )
-    blob.upload_blob(data)
+    blob.upload_blob(data, overwrite=overwrite)
     return self.BASE_URL + blob_name
 
-  def upload_file(self, path: str | os.PathLike, blob_name: str) -> str:
+  def upload_file(self, path: str | os.PathLike, blob_name: str, overwrite=False) -> str:
     with open(path, "rb") as f:
-      return self.upload_bytes(f, blob_name)
+      return self.upload_bytes(f, blob_name, overwrite)