use azure token

.github/workflows/selfdrive_tests.yaml

@@ -317,12 +317,12 @@ jobs:
                         FILEREADER_CACHE=1 CI=1 coverage run selfdrive/test/process_replay/test_processes.py --save-logs && \
                         coverage xml"
     - name: Upload reference logs
-      if: ${{ failure() }}
+      if: ${{ failure() && github.event_name == 'pull_request' && github.repository == 'commaai/openpilot' }}
       run: |
         ${{ env.RUN }} "scons -j$(nproc) && \
                         apt-get install azure-cli && \
                         pip install azure && \
-                        FILEREADER_CACHE=1 CI=1 python selfdrive/test/process_replay/update_refs.py --only-upload"
+                        FILEREADER_CACHE=1 CI=1 AZURE_TOKEN=${{AZURE_COMMADATACI_OPENPILOTCI_TOKEN}} python selfdrive/test/process_replay/update_refs.py --upload-only"
     - name: "Upload coverage to Codecov"
       uses: codecov/codecov-action@v2
     - name: Print diff

selfdrive/test/openpilotci.py

@@ -4,19 +4,22 @@ import sys
 import subprocess
 
 BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/"
-
 TOKEN_PATH = "/data/azure_token"
 
+
 def get_url(route_name, segment_num, log_type="rlog"):
   ext = "hevc" if log_type.endswith('camera') else "bz2"
   return BASE_URL + f"{route_name.replace('|', '/')}/{segment_num}/{log_type}.{ext}"
 
+
 def upload_file(path, name):
   from azure.storage.blob import BlockBlobService  # pylint: disable=import-error
 
   sas_token = None
   if os.path.isfile(TOKEN_PATH):
     sas_token = open(TOKEN_PATH).read().strip()
+  elif "AZURE_TOKEN" in os.environ:
+    sas_token = os.environ["AZURE_TOKEN"]
 
   if sas_token is None:
     sas_token = subprocess.check_output("az storage container generate-sas --account-name commadataci --name openpilotci --https-only --permissions lrw \
@@ -25,6 +28,7 @@ def upload_file(path, name):
   service.create_blob_from_path("openpilotci", name, path)
   return "https://commadataci.blob.core.windows.net/openpilotci/" + name
 
+
 if __name__ == "__main__":
   for f in sys.argv[1:]:
     name = os.path.basename(f)

selfdrive/test/process_replay/update_refs.py

@@ -14,9 +14,9 @@ if __name__ == "__main__":
   parser = argparse.ArgumentParser(description="Updates the reference logs for the current commit")
 
   parser.add_argument("--no-upload", action="store_true")
-  parser.add_argument("--only-upload", action="store_true")  # TODO: split this out into own file upload_refs?
+  parser.add_argument("--upload-only", action="store_true")  # TODO: split this out into own file upload_refs?
   args = parser.parse_args()
-  assert args.no_upload != args.only_upload or not args.no_upload, "Both upload args can't be set"
+  assert args.no_upload != args.upload_only or not args.no_upload, "Both upload args can't be set"
 
   process_replay_dir = os.path.dirname(os.path.abspath(__file__))
   ref_commit_fn = os.path.join(process_replay_dir, "ref_commit")
@@ -28,12 +28,10 @@ if __name__ == "__main__":
     f.write(ref_commit)
 
   for car_brand, segment in segments:
-    if args.only_upload:
+    if args.upload_only:
       for cfg in CONFIGS:
         log_fn = os.path.join(process_replay_dir, f"{segment}_{cfg.proc_name}_{ref_commit}.bz2")
-        if not os.path.exists(log_fn):
-          raise Exception("couldn't find file for uploading: {}".format(log_fn))
-        # upload_file(log_fn, os.path.basename(log_fn))
+        upload_file(log_fn, os.path.basename(log_fn))
         os.remove(log_fn)
         print('Uploaded {}'.format(log_fn))
       continue
@@ -48,6 +46,6 @@ if __name__ == "__main__":
 
       if not args.no_upload:
         upload_file(log_fn, os.path.basename(log_fn))
-        os.remove(log_fn)
+        # os.remove(log_fn)
 
   print("Updated reference logs for commit: {}".format(ref_commit))