diff --git a/poetry.lock b/poetry.lock index 60e1de7266..c5d7b0e28f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -313,47 +313,60 @@ files = [ ] [[package]] -name = "azure-common" -version = "1.1.28" -description = "Microsoft Azure Client Library for Python (Common)" +name = "azure-core" +version = "1.29.3" +description = "Microsoft Azure Core Library for Python" optional = false -python-versions = "*" +python-versions = ">=3.7" files = [ - {file = "azure-common-1.1.28.zip", hash = "sha256:4ac0cd3214e36b6a1b6a442686722a5d8cc449603aa833f3f0f40bda836704a3"}, - {file = "azure_common-1.1.28-py2.py3-none-any.whl", hash = "sha256:5c12d3dcf4ec20599ca6b0d3e09e86e146353d443e7fcc050c9a19c1f9df20ad"}, + {file = "azure-core-1.29.3.tar.gz", hash = "sha256:c92700af982e71c8c73de9f4c20da8b3f03ce2c22d13066e4d416b4629c87903"}, + {file = "azure_core-1.29.3-py3-none-any.whl", hash = "sha256:f8b2910f92b66293d93bd00564924ad20ad48f4a1e150577cf18d1e7d4f9263c"}, ] +[package.dependencies] +requests = ">=2.18.4" +six = ">=1.11.0" +typing-extensions = ">=4.6.0" + +[package.extras] +aio = ["aiohttp (>=3.0)"] + [[package]] -name = "azure-storage-blob" -version = "2.1.0" -description = "Microsoft Azure Storage Blob Client Library for Python" +name = "azure-identity" +version = "1.14.0" +description = "Microsoft Azure Identity Library for Python" optional = false -python-versions = "*" +python-versions = ">=3.7" files = [ - {file = "azure-storage-blob-2.1.0.tar.gz", hash = "sha256:b90323aad60f207f9f90a0c4cf94c10acc313c20b39403398dfba51f25f7b454"}, - {file = "azure_storage_blob-2.1.0-py2.py3-none-any.whl", hash = "sha256:a8e91a51d4f62d11127c7fd8ba0077385c5b11022f0269f8a2a71b9fc36bef31"}, + {file = "azure-identity-1.14.0.zip", hash = "sha256:72441799f8c5c89bfe21026965e266672a7c5d050c2c65119ef899dd5362e2b1"}, + {file = "azure_identity-1.14.0-py3-none-any.whl", hash = "sha256:edabf0e010eb85760e1dd19424d5e8f97ba2c9caff73a16e7b30ccbdbcce369b"}, ] [package.dependencies] -azure-common = ">=1.1.5" -azure-storage-common = ">=2.1,<3.0" +azure-core = ">=1.11.0,<2.0.0" +cryptography = ">=2.5" +msal = ">=1.20.0,<2.0.0" +msal-extensions = ">=0.3.0,<2.0.0" [[package]] -name = "azure-storage-common" -version = "2.1.0" -description = "Microsoft Azure Storage Common Client Library for Python" +name = "azure-storage-blob" +version = "12.17.0" +description = "Microsoft Azure Blob Storage Client Library for Python" optional = false -python-versions = "*" +python-versions = ">=3.7" files = [ - {file = "azure-storage-common-2.1.0.tar.gz", hash = "sha256:ccedef5c67227bc4d6670ffd37cec18fb529a1b7c3a5e53e4096eb0cf23dc73f"}, - {file = "azure_storage_common-2.1.0-py2.py3-none-any.whl", hash = "sha256:b01a491a18839b9d05a4fe3421458a0ddb5ab9443c14e487f40d16f9a1dc2fbe"}, + {file = "azure-storage-blob-12.17.0.zip", hash = "sha256:c14b785a17050b30fc326a315bdae6bc4a078855f4f94a4c303ad74a48dc8c63"}, + {file = "azure_storage_blob-12.17.0-py3-none-any.whl", hash = "sha256:0016e0c549a80282d7b4920c03f2f4ba35c53e6e3c7dbcd2a4a8c8eb3882c1e7"}, ] [package.dependencies] -azure-common = ">=1.1.5" -cryptography = "*" -python-dateutil = "*" -requests = "*" +azure-core = ">=1.28.0,<2.0.0" +cryptography = ">=2.1.4" +isodate = ">=0.6.1" +typing-extensions = ">=4.3.0" + +[package.extras] +aio = ["azure-core[aio] (>=1.28.0,<2.0.0)"] [[package]] name = "babel" @@ -1524,6 +1537,20 @@ files = [ {file = "ioctl-opt-1.3.tar.gz", hash = "sha256:5ed4f9a80d2e02e152a43d3648d7ed8821a0aac5ea88ecc5fcc14460ff7cf2f9"}, ] +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + [[package]] name = "jinja2" version = "3.1.2" @@ -1942,6 +1969,43 @@ docs = ["sphinx"] gmpy = ["gmpy2 (>=2.1.0a4)"] tests = ["pytest (>=4.6)"] +[[package]] +name = "msal" +version = "1.23.0" +description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." +optional = false +python-versions = "*" +files = [ + {file = "msal-1.23.0-py2.py3-none-any.whl", hash = "sha256:3342e0837a047007f9d479e814b559c3219767453d57920dc40a31986862048b"}, + {file = "msal-1.23.0.tar.gz", hash = "sha256:25c9a33acf84301f93d1fdbe9f1a9c60cd38af0d5fffdbfa378138fc7bc1e86b"}, +] + +[package.dependencies] +cryptography = ">=0.6,<44" +PyJWT = {version = ">=1.0.0,<3", extras = ["crypto"]} +requests = ">=2.0.0,<3" + +[package.extras] +broker = ["pymsalruntime (>=0.13.2,<0.14)"] + +[[package]] +name = "msal-extensions" +version = "1.0.0" +description = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism." +optional = false +python-versions = "*" +files = [ + {file = "msal-extensions-1.0.0.tar.gz", hash = "sha256:c676aba56b0cce3783de1b5c5ecfe828db998167875126ca4b47dc6436451354"}, + {file = "msal_extensions-1.0.0-py2.py3-none-any.whl", hash = "sha256:91e3db9620b822d0ed2b4d1850056a0f133cba04455e62f11612e40f5502f2ee"}, +] + +[package.dependencies] +msal = ">=0.4.1,<2.0.0" +portalocker = [ + {version = ">=1.0,<3", markers = "python_version >= \"3.5\" and platform_system != \"Windows\""}, + {version = ">=1.6,<3", markers = "python_version >= \"3.5\" and platform_system == \"Windows\""}, +] + [[package]] name = "multidict" version = "6.0.4" @@ -2553,6 +2617,25 @@ files = [ dev = ["pylint (>=2.15.10,<2.16.0)", "pytest (>=7.0,<8.0)", "pytest-cov (>=4.0,<5.0)", "sphinx (>=4.2.0,<4.3.0)", "sphinx-rtd-theme (>=1.0.0,<1.1.0)", "toml (>=0.10.2,<0.11.0)"] publish = ["build (>=0.8,<1.0)", "twine (>=4.0,<5.0)"] +[[package]] +name = "portalocker" +version = "2.7.0" +description = "Wraps the portalocker recipe for easy usage" +optional = false +python-versions = ">=3.5" +files = [ + {file = "portalocker-2.7.0-py2.py3-none-any.whl", hash = "sha256:a07c5b4f3985c3cf4798369631fb7011adb498e2a46d8440efc75a8f29a0f983"}, + {file = "portalocker-2.7.0.tar.gz", hash = "sha256:032e81d534a88ec1736d03f780ba073f047a06c478b06e2937486f334e955c51"}, +] + +[package.dependencies] +pywin32 = {version = ">=226", markers = "platform_system == \"Windows\""} + +[package.extras] +docs = ["sphinx (>=1.7.1)"] +redis = ["redis"] +tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "pytest-timeout (>=2.1.0)", "redis", "sphinx (>=6.0.0)"] + [[package]] name = "pprofile" version = "2.1.0" @@ -2871,6 +2954,9 @@ files = [ {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, ] +[package.dependencies] +cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} + [package.extras] crypto = ["cryptography (>=3.4.0)"] dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] @@ -3219,6 +3305,29 @@ files = [ {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, ] +[[package]] +name = "pywin32" +version = "306" +description = "Python for Window Extensions" +optional = false +python-versions = "*" +files = [ + {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, + {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, + {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, + {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, + {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, + {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, + {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, + {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, + {file = "pywin32-306-cp37-cp37m-win32.whl", hash = "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65"}, + {file = "pywin32-306-cp37-cp37m-win_amd64.whl", hash = "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36"}, + {file = "pywin32-306-cp38-cp38-win32.whl", hash = "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a"}, + {file = "pywin32-306-cp38-cp38-win_amd64.whl", hash = "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0"}, + {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"}, + {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, +] + [[package]] name = "pyyaml" version = "6.0.1" @@ -4213,4 +4322,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "~3.11" -content-hash = "320aa5cc075d746403abb872211ebdaf55e9504face8dab8c9b270c71953675d" +content-hash = "7e2bfde2719e7d7bb4b1627b0657e9ab4a9f4e1637d8b8ae6d5c80c7861e2052" diff --git a/pyproject.toml b/pyproject.toml index b5d03e9b4d..e350f070c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,7 +96,8 @@ sconscontrib = {git = "https://github.com/SCons/scons-contrib.git"} [tool.poetry.group.dev.dependencies] av = "*" -azure-storage-blob = "~2.1" +azure-identity = "*" +azure-storage-blob = "*" breathe = "*" carla = { url = "https://github.com/commaai/carla/releases/download/3.11.4/carla-0.9.14-cp311-cp311-linux_x86_64.whl", platform = "linux", markers = "platform_machine == 'x86_64'" } coverage = "*" diff --git a/selfdrive/test/openpilotci.py b/selfdrive/test/openpilotci.py index fe1a0b0335..f590b66e54 100755 --- a/selfdrive/test/openpilotci.py +++ b/selfdrive/test/openpilotci.py @@ -1,43 +1,64 @@ #!/usr/bin/env python3 import os -import sys -import subprocess +from datetime import datetime, timedelta +from functools import lru_cache +from pathlib import Path +from typing import IO, Union -BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/" -TOKEN_PATH = "/data/azure_token" +DATA_CI_ACCOUNT = "commadataci" +DATA_CI_ACCOUNT_URL = f"https://{DATA_CI_ACCOUNT}.blob.core.windows.net" +DATA_CI_CONTAINER = "openpilotci" +BASE_URL = f"{DATA_CI_ACCOUNT_URL}/{DATA_CI_CONTAINER}/" +TOKEN_PATH = Path("/data/azure_token") -def get_url(route_name, segment_num, log_type="rlog"): + +def get_url(route_name: str, segment_num, log_type="rlog") -> str: ext = "hevc" if log_type.endswith('camera') else "bz2" return BASE_URL + f"{route_name.replace('|', '/')}/{segment_num}/{log_type}.{ext}" -def get_sas_token(): - sas_token = os.environ.get("AZURE_TOKEN", None) - if os.path.isfile(TOKEN_PATH): - sas_token = open(TOKEN_PATH).read().strip() - if sas_token is None: - sas_token = subprocess.check_output("az storage container generate-sas --account-name commadataci --name openpilotci \ - --https-only --permissions lrw --expiry $(date -u '+%Y-%m-%dT%H:%M:%SZ' -d '+1 hour') \ - --auth-mode login --as-user --output tsv", shell=True).decode().strip("\n") +@lru_cache +def get_azure_credential(): + if "AZURE_TOKEN" in os.environ: + return os.environ["AZURE_TOKEN"] + elif TOKEN_PATH.is_file(): + return TOKEN_PATH.read_text().strip() + else: + from azure.identity import AzureCliCredential + return AzureCliCredential() + - return sas_token +@lru_cache +def get_container_sas(account_name: str, container_name: str): + from azure.storage.blob import BlobServiceClient, ContainerSasPermissions, generate_container_sas + start_time = datetime.utcnow() + expiry_time = start_time + timedelta(hours=1) + blob_service = BlobServiceClient( + account_url=f"https://{account_name}.blob.core.windows.net", + credential=get_azure_credential(), + ) + return generate_container_sas( + account_name, + container_name, + user_delegation_key=blob_service.get_user_delegation_key(start_time, expiry_time), + permission=ContainerSasPermissions(read=True, write=True, list=True), + expiry=expiry_time, + ) -def upload_bytes(data, name): - from azure.storage.blob import BlockBlobService - service = BlockBlobService(account_name="commadataci", sas_token=get_sas_token()) - service.create_blob_from_bytes("openpilotci", name, data) - return BASE_URL + name -def upload_file(path, name): - from azure.storage.blob import BlockBlobService - service = BlockBlobService(account_name="commadataci", sas_token=get_sas_token()) - service.create_blob_from_path("openpilotci", name, path) - return BASE_URL + name +def upload_bytes(data: Union[bytes, IO], blob_name: str) -> str: + from azure.storage.blob import BlobClient + blob = BlobClient( + account_url=DATA_CI_ACCOUNT_URL, + container_name=DATA_CI_CONTAINER, + blob_name=blob_name, + credential=get_azure_credential(), + ) + blob.upload_blob(data) + return BASE_URL + blob_name -if __name__ == "__main__": - for f in sys.argv[1:]: - name = os.path.basename(f) - url = upload_file(f, name) - print(url) +def upload_file(path: Union[str, os.PathLike], blob_name: str) -> str: + with open(path, "rb") as f: + return upload_bytes(f, blob_name) diff --git a/selfdrive/test/update_ci_routes.py b/selfdrive/test/update_ci_routes.py index dda8568fa8..8157066334 100755 --- a/selfdrive/test/update_ci_routes.py +++ b/selfdrive/test/update_ci_routes.py @@ -1,32 +1,38 @@ #!/usr/bin/env python3 -from functools import lru_cache -import sys import subprocess +import sys +from functools import lru_cache +from typing import Iterable, Optional + +from azure.storage.blob import ContainerClient from tqdm import tqdm -from azure.storage.blob import BlockBlobService from openpilot.selfdrive.car.tests.routes import routes as test_car_models_routes from openpilot.selfdrive.locationd.test.test_laikad import UBLOX_TEST_ROUTE, QCOM_TEST_ROUTE from openpilot.selfdrive.test.process_replay.test_processes import source_segments as replay_segments -from xx.chffr.lib import azureutil -from xx.chffr.lib.storage import _DATA_ACCOUNT_PRODUCTION, _DATA_ACCOUNT_CI, _DATA_BUCKET_PRODUCTION +from openpilot.selfdrive.test.openpilotci import (DATA_CI_ACCOUNT, DATA_CI_ACCOUNT_URL, DATA_CI_CONTAINER, + get_azure_credential, get_container_sas) + +DATA_PROD_ACCOUNT = "commadata2" +DATA_PROD_CONTAINER = "commadata2" SOURCES = [ - (_DATA_ACCOUNT_PRODUCTION, _DATA_BUCKET_PRODUCTION), - (_DATA_ACCOUNT_CI, "commadataci"), + (DATA_PROD_ACCOUNT, DATA_PROD_CONTAINER), + (DATA_CI_ACCOUNT, DATA_CI_CONTAINER), ] @lru_cache def get_azure_keys(): - dest_key = azureutil.get_user_token(_DATA_ACCOUNT_CI, "openpilotci") - source_keys = [azureutil.get_user_token(account, bucket) for account, bucket in SOURCES] - service = BlockBlobService(_DATA_ACCOUNT_CI, sas_token=dest_key) - return dest_key, source_keys, service + dest_container = ContainerClient(DATA_CI_ACCOUNT_URL, DATA_CI_CONTAINER, credential=get_azure_credential()) + dest_key = get_container_sas(DATA_CI_ACCOUNT, DATA_CI_CONTAINER) + source_keys = [get_container_sas(*s) for s in SOURCES] + return dest_container, dest_key, source_keys -def upload_route(path, exclude_patterns=None): - dest_key, _, _ = get_azure_keys() +def upload_route(path: str, exclude_patterns: Optional[Iterable[str]] = None) -> None: + # TODO: use azure-storage-blob instead of azcopy, simplifies auth + dest_key = get_container_sas(DATA_CI_ACCOUNT, DATA_CI_CONTAINER) if exclude_patterns is None: exclude_patterns = ['*/dcamera.hevc'] @@ -37,28 +43,30 @@ def upload_route(path, exclude_patterns=None): "azcopy", "copy", f"{path}/*", - f"https://{_DATA_ACCOUNT_CI}.blob.core.windows.net/openpilotci/{destpath}?{dest_key}", + f"https://{DATA_CI_ACCOUNT}.blob.core.windows.net/{DATA_CI_CONTAINER}/{destpath}?{dest_key}", "--recursive=false", "--overwrite=false", ] + [f"--exclude-pattern={p}" for p in exclude_patterns] subprocess.check_call(cmd) -def sync_to_ci_public(route): - dest_key, source_keys, service = get_azure_keys() + +def sync_to_ci_public(route: str) -> bool: + dest_container, dest_key, source_keys = get_azure_keys() key_prefix = route.replace('|', '/') dongle_id = key_prefix.split('/')[0] - if next(azureutil.list_all_blobs(service, "openpilotci", prefix=key_prefix), None) is not None: + if next(dest_container.list_blob_names(name_starts_with=key_prefix), None) is not None: return True print(f"Uploading {route}") for (source_account, source_bucket), source_key in zip(SOURCES, source_keys, strict=True): + # assumes az login has been run print(f"Trying {source_account}/{source_bucket}") cmd = [ "azcopy", "copy", f"https://{source_account}.blob.core.windows.net/{source_bucket}/{key_prefix}?{source_key}", - f"https://{_DATA_ACCOUNT_CI}.blob.core.windows.net/openpilotci/{dongle_id}?{dest_key}", + f"https://{DATA_CI_ACCOUNT}.blob.core.windows.net/{DATA_CI_CONTAINER}/{dongle_id}?{dest_key}", "--recursive=true", "--overwrite=false", "--exclude-pattern=*/dcamera.hevc",