From cba0e3ad426d85faeb17110004d7f4fc847e25cf Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Wed, 4 May 2022 13:20:42 -0700 Subject: [PATCH] use azure token --- .github/workflows/selfdrive_tests.yaml | 4 ++-- selfdrive/test/openpilotci.py | 6 +++++- selfdrive/test/process_replay/update_refs.py | 12 +++++------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index db490dd6a6..dcc89d33c5 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -317,12 +317,12 @@ jobs: FILEREADER_CACHE=1 CI=1 coverage run selfdrive/test/process_replay/test_processes.py --save-logs && \ coverage xml" - name: Upload reference logs - if: ${{ failure() }} + if: ${{ failure() && github.event_name == 'pull_request' && github.repository == 'commaai/openpilot' }} run: | ${{ env.RUN }} "scons -j$(nproc) && \ apt-get install azure-cli && \ pip install azure && \ - FILEREADER_CACHE=1 CI=1 python selfdrive/test/process_replay/update_refs.py --only-upload" + FILEREADER_CACHE=1 CI=1 AZURE_TOKEN=${{AZURE_COMMADATACI_OPENPILOTCI_TOKEN}} python selfdrive/test/process_replay/update_refs.py --upload-only" - name: "Upload coverage to Codecov" uses: codecov/codecov-action@v2 - name: Print diff diff --git a/selfdrive/test/openpilotci.py b/selfdrive/test/openpilotci.py index 6483b6717f..105f0b0911 100755 --- a/selfdrive/test/openpilotci.py +++ b/selfdrive/test/openpilotci.py @@ -4,19 +4,22 @@ import sys import subprocess BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/" - TOKEN_PATH = "/data/azure_token" + def get_url(route_name, segment_num, log_type="rlog"): ext = "hevc" if log_type.endswith('camera') else "bz2" return BASE_URL + f"{route_name.replace('|', '/')}/{segment_num}/{log_type}.{ext}" + def upload_file(path, name): from azure.storage.blob import BlockBlobService # pylint: disable=import-error sas_token = None if os.path.isfile(TOKEN_PATH): sas_token = open(TOKEN_PATH).read().strip() + elif "AZURE_TOKEN" in os.environ: + sas_token = os.environ["AZURE_TOKEN"] if sas_token is None: sas_token = subprocess.check_output("az storage container generate-sas --account-name commadataci --name openpilotci --https-only --permissions lrw \ @@ -25,6 +28,7 @@ def upload_file(path, name): service.create_blob_from_path("openpilotci", name, path) return "https://commadataci.blob.core.windows.net/openpilotci/" + name + if __name__ == "__main__": for f in sys.argv[1:]: name = os.path.basename(f) diff --git a/selfdrive/test/process_replay/update_refs.py b/selfdrive/test/process_replay/update_refs.py index 6bc3a72ae2..975d4c9cb0 100755 --- a/selfdrive/test/process_replay/update_refs.py +++ b/selfdrive/test/process_replay/update_refs.py @@ -14,9 +14,9 @@ if __name__ == "__main__": parser = argparse.ArgumentParser(description="Updates the reference logs for the current commit") parser.add_argument("--no-upload", action="store_true") - parser.add_argument("--only-upload", action="store_true") # TODO: split this out into own file upload_refs? + parser.add_argument("--upload-only", action="store_true") # TODO: split this out into own file upload_refs? args = parser.parse_args() - assert args.no_upload != args.only_upload or not args.no_upload, "Both upload args can't be set" + assert args.no_upload != args.upload_only or not args.no_upload, "Both upload args can't be set" process_replay_dir = os.path.dirname(os.path.abspath(__file__)) ref_commit_fn = os.path.join(process_replay_dir, "ref_commit") @@ -28,12 +28,10 @@ if __name__ == "__main__": f.write(ref_commit) for car_brand, segment in segments: - if args.only_upload: + if args.upload_only: for cfg in CONFIGS: log_fn = os.path.join(process_replay_dir, f"{segment}_{cfg.proc_name}_{ref_commit}.bz2") - if not os.path.exists(log_fn): - raise Exception("couldn't find file for uploading: {}".format(log_fn)) - # upload_file(log_fn, os.path.basename(log_fn)) + upload_file(log_fn, os.path.basename(log_fn)) os.remove(log_fn) print('Uploaded {}'.format(log_fn)) continue @@ -48,6 +46,6 @@ if __name__ == "__main__": if not args.no_upload: upload_file(log_fn, os.path.basename(log_fn)) - os.remove(log_fn) + # os.remove(log_fn) print("Updated reference logs for commit: {}".format(ref_commit))