From 01fcd7b72722af6bbe42ac011409d09bcc9599d4 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Mon, 6 May 2024 16:42:35 -0700 Subject: [PATCH] revert changes to release/ (#32364) * revert changes to release/ * put those back * revert Jenkinsfile old-commit-hash: 1c42b8a05fc8b525902904a855ff04af8f56d869 --- Jenkinsfile | 219 ++++++++++++-------------------- release/README.md | 34 ----- release/build_git_release.sh | 105 --------------- release/build_release.sh | 80 ++++++++++-- release/package_casync_agnos.py | 59 --------- release/package_casync_build.py | 108 ---------------- 6 files changed, 147 insertions(+), 458 deletions(-) delete mode 100644 release/README.md delete mode 100755 release/build_git_release.sh delete mode 100755 release/package_casync_agnos.py delete mode 100755 release/package_casync_build.py diff --git a/Jenkinsfile b/Jenkinsfile index efba905c6b..f21d7f20fe 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,11 +9,6 @@ def retryWithDelay(int maxRetries, int delay, Closure body) { throw Exception("Failed after ${maxRetries} retries") } -// check if started by timer: https://gist.github.com/aaclarker/75b8a0eb2b4d600779f84f8e849f2c37 -def isJobStartedByTimer() { - return currentBuild.getBuildCauses()[0]["shortDescription"].matches("Started by timer"); -} - def device(String ip, String step_label, String cmd) { withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) { def ssh_cmd = """ @@ -31,7 +26,6 @@ export SOURCE_DIR=${env.SOURCE_DIR} export GIT_BRANCH=${env.GIT_BRANCH} export GIT_COMMIT=${env.GIT_COMMIT} export AZURE_TOKEN='${env.AZURE_TOKEN}' -export AZURE_TOKEN_OPENPILOT_RELEASES='${env.AZURE_TOKEN_OPENPILOT_RELEASES}' export MAPBOX_TOKEN='${env.MAPBOX_TOKEN}' # only use 1 thread for tici tests since most require HIL export PYTEST_ADDOPTS="-n 0" @@ -111,7 +105,7 @@ def pcStage(String stageName, Closure body) { checkout scm - def dockerArgs = "--user=batman -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/scons_cache:/tmp/scons_cache -e PYTHONPATH=${env.WORKSPACE} -e AZURE_TOKEN_OPENPILOT_RELEASES='${env.AZURE_TOKEN_OPENPILOT_RELEASES}' --cpus=8 --memory 16g -e PYTEST_ADDOPTS='-n8'"; + def dockerArgs = "--user=batman -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/scons_cache:/tmp/scons_cache -e PYTHONPATH=${env.WORKSPACE} --cpus=8 --memory 16g -e PYTEST_ADDOPTS='-n8'"; def openpilot_base = retryWithDelay (3, 15) { return docker.build("openpilot-base:build-${env.GIT_COMMIT}", "-f Dockerfile.openpilot_base .") @@ -119,7 +113,7 @@ def pcStage(String stageName, Closure body) { lock(resource: "", label: 'pc', inversePrecedence: true, quantity: 1) { openpilot_base.inside(dockerArgs) { - timeout(time: 25, unit: 'MINUTES') { + timeout(time: 20, unit: 'MINUTES') { try { retryWithDelay (3, 15) { sh "git config --global --add safe.directory '*'" @@ -141,56 +135,14 @@ def pcStage(String stageName, Closure body) { def setupCredentials() { withCredentials([ string(credentialsId: 'azure_token', variable: 'AZURE_TOKEN'), - string(credentialsId: 'azure_token_openpilot_releases', variable: 'AZURE_TOKEN_OPENPILOT_RELEASES'), string(credentialsId: 'mapbox_token', variable: 'MAPBOX_TOKEN') ]) { env.AZURE_TOKEN = "${AZURE_TOKEN}" - env.AZURE_TOKEN_OPENPILOT_RELEASES = "${AZURE_TOKEN_OPENPILOT_RELEASES}" env.MAPBOX_TOKEN = "${MAPBOX_TOKEN}" } } -def build_git_release(String channel_name) { - return parallel ( - "${channel_name} (git)": { - deviceStage("build git", "tici-needs-can", [], [ - ["build ${channel_name}", "RELEASE_BRANCH=${channel_name} $SOURCE_DIR/release/build_git_release.sh"], - ]) - } - ) -} - - -def build_casync_release(String channel_name, def is_release) { - def extra_env = is_release ? "RELEASE=1 " : "" - def build_dir = "/data/openpilot" - - extra_env += "TMPDIR=/data/tmp PYTHONPATH=$SOURCE_DIR" - - return deviceStage("build casync", "tici-needs-can", [], [ - ["build", "${extra_env} $SOURCE_DIR/release/build_release.sh ${build_dir}"], - ["package + upload", "${extra_env} $SOURCE_DIR/release/package_casync_build.py ${build_dir}"], - ]) -} - - -def build_stage() { - return parallel ( - 'nightly': { - build_casync_release("nightly", true); - }, - 'master': { - build_casync_release("master", false); - }, - 'publish agnos': { - pcStage("publish agnos") { - sh "PYTHONWARNINGS=ignore release/package_casync_agnos.py" - } - } - ) -} - node { env.CI = "1" env.PYTHONWARNINGS = "error" @@ -205,105 +157,94 @@ node { 'testing-closet*', 'hotfix-*'] def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*') - def nightlyBranch = "master" - - def props = []; - - if (env.BRANCH_NAME == nightlyBranch) { - props.add(pipelineTriggers([ - cron('0 9 * * *') // at 2am PST (9am UTC) every night - ])) - } - - if (env.BRANCH_NAME != "master") { - props.add(disableConcurrentBuilds(abortPrevious: true)) + if (env.BRANCH_NAME != 'master') { + properties([ + disableConcurrentBuilds(abortPrevious: true) + ]) } - properties(props); - try { if (env.BRANCH_NAME == 'devel-staging') { - build_git_release("release3-staging") + deviceStage("build release3-staging", "tici-needs-can", [], [ + ["build release3-staging", "RELEASE_BRANCH=release3-staging $SOURCE_DIR/release/build_release.sh"], + ]) } if (env.BRANCH_NAME == 'master-ci') { - build_git_release("nightly") + deviceStage("build nightly", "tici-needs-can", [], [ + ["build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"], + ]) } if (!env.BRANCH_NAME.matches(excludeRegex)) { - parallel ( - // tici tests - 'onroad tests': { - deviceStage("onroad", "tici-needs-can", [], [ - // TODO: ideally, this test runs in master-ci, but it takes 5+m to build it - //["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR $SOURCE_DIR/scripts/retry.sh ./build_devel.sh"], - ["build openpilot", "cd selfdrive/manager && ./build.py"], - ["check dirty", "release/check-dirty.sh"], - ["onroad tests", "pytest selfdrive/test/test_onroad.py -s"], - ["time to onroad", "pytest selfdrive/test/test_time_to_onroad.py"], - ]) - }, - 'HW + Unit Tests': { - deviceStage("tici-hardware", "tici-common", ["UNSAFE=1"], [ - ["build", "cd selfdrive/manager && ./build.py"], - ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"], - ["test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"], - ["test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py"], - ["test pigeond", "pytest system/ubloxd/tests/test_pigeond.py"], - ["test manager", "pytest selfdrive/manager/test/test_manager.py"], - ]) - }, - 'loopback': { - deviceStage("loopback", "tici-loopback", ["UNSAFE=1"], [ - ["build openpilot", "cd selfdrive/manager && ./build.py"], - ["test boardd loopback", "pytest selfdrive/boardd/tests/test_boardd_loopback.py"], - ]) - }, - 'camerad': { - deviceStage("AR0231", "tici-ar0231", ["UNSAFE=1"], [ - ["build", "cd selfdrive/manager && ./build.py"], - ["test camerad", "pytest system/camerad/test/test_camerad.py"], - ["test exposure", "pytest system/camerad/test/test_exposure.py"], - ]) - deviceStage("OX03C10", "tici-ox03c10", ["UNSAFE=1"], [ - ["build", "cd selfdrive/manager && ./build.py"], - ["test camerad", "pytest system/camerad/test/test_camerad.py"], - ["test exposure", "pytest system/camerad/test/test_exposure.py"], - ]) - }, - 'sensord': { - deviceStage("LSM + MMC", "tici-lsmc", ["UNSAFE=1"], [ - ["build", "cd selfdrive/manager && ./build.py"], - ["test sensord", "pytest system/sensord/tests/test_sensord.py"], - ]) - deviceStage("BMX + LSM", "tici-bmx-lsm", ["UNSAFE=1"], [ - ["build", "cd selfdrive/manager && ./build.py"], - ["test sensord", "pytest system/sensord/tests/test_sensord.py"], - ]) - }, - 'replay': { - deviceStage("model-replay", "tici-replay", ["UNSAFE=1"], [ - ["build", "cd selfdrive/manager && ./build.py"], - ["model replay", "selfdrive/test/process_replay/model_replay.py"], - ]) - }, - 'tizi': { - deviceStage("tizi", "tizi", ["UNSAFE=1"], [ - ["build openpilot", "cd selfdrive/manager && ./build.py"], - ["test boardd loopback", "SINGLE_PANDA=1 pytest selfdrive/boardd/tests/test_boardd_loopback.py"], - ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"], - ["test amp", "pytest system/hardware/tici/tests/test_amplifier.py"], - ["test hw", "pytest system/hardware/tici/tests/test_hardware.py"], - ["test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py"], - ]) - }, - ) - } - - if (env.BRANCH_NAME == nightlyBranch && isJobStartedByTimer()) { - stage('build release') { - build_stage() - } + parallel ( + // tici tests + 'onroad tests': { + deviceStage("onroad", "tici-needs-can", [], [ + // TODO: ideally, this test runs in master-ci, but it takes 5+m to build it + //["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR $SOURCE_DIR/scripts/retry.sh ./build_devel.sh"], + ["build openpilot", "cd selfdrive/manager && ./build.py"], + ["check dirty", "release/check-dirty.sh"], + ["onroad tests", "pytest selfdrive/test/test_onroad.py -s"], + ["time to onroad", "pytest selfdrive/test/test_time_to_onroad.py"], + ]) + }, + 'HW + Unit Tests': { + deviceStage("tici-hardware", "tici-common", ["UNSAFE=1"], [ + ["build", "cd selfdrive/manager && ./build.py"], + ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"], + ["test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"], + ["test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py"], + ["test pigeond", "pytest system/sensord/tests/test_pigeond.py"], + ["test manager", "pytest selfdrive/manager/test/test_manager.py"], + ]) + }, + 'loopback': { + deviceStage("loopback", "tici-loopback", ["UNSAFE=1"], [ + ["build openpilot", "cd selfdrive/manager && ./build.py"], + ["test boardd loopback", "pytest selfdrive/boardd/tests/test_boardd_loopback.py"], + ]) + }, + 'camerad': { + deviceStage("AR0231", "tici-ar0231", ["UNSAFE=1"], [ + ["build", "cd selfdrive/manager && ./build.py"], + ["test camerad", "pytest system/camerad/test/test_camerad.py"], + ["test exposure", "pytest system/camerad/test/test_exposure.py"], + ]) + deviceStage("OX03C10", "tici-ox03c10", ["UNSAFE=1"], [ + ["build", "cd selfdrive/manager && ./build.py"], + ["test camerad", "pytest system/camerad/test/test_camerad.py"], + ["test exposure", "pytest system/camerad/test/test_exposure.py"], + ]) + }, + 'sensord': { + deviceStage("LSM + MMC", "tici-lsmc", ["UNSAFE=1"], [ + ["build", "cd selfdrive/manager && ./build.py"], + ["test sensord", "pytest system/sensord/tests/test_sensord.py"], + ]) + deviceStage("BMX + LSM", "tici-bmx-lsm", ["UNSAFE=1"], [ + ["build", "cd selfdrive/manager && ./build.py"], + ["test sensord", "pytest system/sensord/tests/test_sensord.py"], + ]) + }, + 'replay': { + deviceStage("model-replay", "tici-replay", ["UNSAFE=1"], [ + ["build", "cd selfdrive/manager && ./build.py"], + ["model replay", "selfdrive/test/process_replay/model_replay.py"], + ]) + }, + 'tizi': { + deviceStage("tizi", "tizi", ["UNSAFE=1"], [ + ["build openpilot", "cd selfdrive/manager && ./build.py"], + ["test boardd loopback", "SINGLE_PANDA=1 pytest selfdrive/boardd/tests/test_boardd_loopback.py"], + ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"], + ["test amp", "pytest system/hardware/tici/tests/test_amplifier.py"], + ["test hw", "pytest system/hardware/tici/tests/test_hardware.py"], + ["test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py"], + ]) + }, + + ) } } catch (Exception e) { currentBuild.result = 'FAILED' diff --git a/release/README.md b/release/README.md deleted file mode 100644 index 89e6cce6f3..0000000000 --- a/release/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# openpilot releases - - -## terms - -- `channel` - a named version of openpilot (git branch, casync caibx) which receives updates -- `build` - a copy of openpilot ready for distribution, already built for a specific device -- `build_style` - type of build, either `debug` or `release` - - `debug` - build with `ALLOW_DEBUG=true`, can test experimental features like longitudinal on alpha cars - - `release` - build with `ALLOW_DEBUG=false`, experimental features disabled - - -## openpilot channels - -| channel | build_style | description | -| ----------- | ----------- | ---------- | -| release | `release` | stable release of openpilot | -| staging | `release` | release candidate of openpilot for final verification | -| nightly | `release` | generated nightly from last commit passing CI tests | -| master | `debug` | current master commit with experimental features enabled | -| git branches | `debug` | installed manually, experimental features enabled, build required | - - -## build - -`release/build_release.sh ` - creates an openpilot build into `build_dir`, ready for distribution - -## packaging a casync release - -`release/package_casync_build.py ` - packages an openpilot build into a casync tar and uploads to `openpilot-releases` - -## release builds - -to create a release build, set `RELEASE=1` environment variable when running the build script diff --git a/release/build_git_release.sh b/release/build_git_release.sh deleted file mode 100755 index 08f8a5a185..0000000000 --- a/release/build_git_release.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/bash -e - -# git diff --name-status origin/release3-staging | grep "^A" | less - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" - -cd $DIR - -BUILD_DIR=/data/openpilot -SOURCE_DIR="$(git rev-parse --show-toplevel)" - -if [ -f /TICI ]; then - FILES_SRC="release/files_tici" -else - echo "no release files set" - exit 1 -fi - -if [ -z "$RELEASE_BRANCH" ]; then - echo "RELEASE_BRANCH is not set" - exit 1 -fi - - -# set git identity -source $DIR/identity.sh - -echo "[-] Setting up repo T=$SECONDS" -rm -rf $BUILD_DIR -mkdir -p $BUILD_DIR -cd $BUILD_DIR -git init -git remote add origin git@github.com:commaai/openpilot.git -git checkout --orphan $RELEASE_BRANCH - -# do the files copy -echo "[-] copying files T=$SECONDS" -cd $SOURCE_DIR -cp -pR --parents $(cat release/files_common) $BUILD_DIR/ -cp -pR --parents $(cat $FILES_SRC) $BUILD_DIR/ - -# in the directory -cd $BUILD_DIR - -rm -f panda/board/obj/panda.bin.signed -rm -f panda/board/obj/panda_h7.bin.signed - -VERSION=$(cat common/version.h | awk -F[\"-] '{print $2}') -echo "#define COMMA_VERSION \"$VERSION-release\"" > common/version.h - -echo "[-] committing version $VERSION T=$SECONDS" -git add -f . -git commit -a -m "openpilot v$VERSION release" - -# Build -export PYTHONPATH="$BUILD_DIR" -scons -j$(nproc) - -# release panda fw -CERT=/data/pandaextra/certs/release RELEASE=1 scons -j$(nproc) panda/ - -# Ensure no submodules in release -if test "$(git submodule--helper list | wc -l)" -gt "0"; then - echo "submodules found:" - git submodule--helper list - exit 1 -fi -git submodule status - -# Cleanup -find . -name '*.a' -delete -find . -name '*.o' -delete -find . -name '*.os' -delete -find . -name '*.pyc' -delete -find . -name 'moc_*' -delete -find . -name '__pycache__' -delete -rm -rf .sconsign.dblite Jenkinsfile release/ -rm selfdrive/modeld/models/supercombo.onnx - -# Restore third_party -git checkout third_party/ - -# Mark as prebuilt release -touch prebuilt - -# Add built files to git -git add -f . -git commit --amend -m "openpilot v$VERSION" - -# Run tests -TEST_FILES="tools/" -cd $SOURCE_DIR -cp -pR -n --parents $TEST_FILES $BUILD_DIR/ -cd $BUILD_DIR -RELEASE=1 selfdrive/test/test_onroad.py -#selfdrive/manager/test/test_manager.py -#selfdrive/car/tests/test_car_interfaces.py -rm -rf $TEST_FILES - -if [ ! -z "$RELEASE_BRANCH" ]; then - echo "[-] pushing release T=$SECONDS" - git push -f origin $RELEASE_BRANCH:$RELEASE_BRANCH -fi - -echo "[-] done T=$SECONDS" diff --git a/release/build_release.sh b/release/build_release.sh index 19c06700fa..fc15cf6cdf 100755 --- a/release/build_release.sh +++ b/release/build_release.sh @@ -1,40 +1,72 @@ -#!/usr/bin/bash +#!/usr/bin/bash -e -set -e +# git diff --name-status origin/release3-staging | grep "^A" | less DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -SOURCE_DIR="$(git -C $DIR rev-parse --show-toplevel)" -BUILD_DIR=${1:-$(mktemp -d)} + +cd $DIR + +BUILD_DIR=/data/openpilot +SOURCE_DIR="$(git rev-parse --show-toplevel)" if [ -f /TICI ]; then FILES_SRC="release/files_tici" else - FILES_SRC="release/files_pc" + echo "no release files set" + exit 1 fi -echo "Building openpilot into $BUILD_DIR" +if [ -z "$RELEASE_BRANCH" ]; then + echo "RELEASE_BRANCH is not set" + exit 1 +fi + + +# set git identity +source $DIR/identity.sh +echo "[-] Setting up repo T=$SECONDS" rm -rf $BUILD_DIR mkdir -p $BUILD_DIR +cd $BUILD_DIR +git init +git remote add origin git@github.com:commaai/openpilot.git +git checkout --orphan $RELEASE_BRANCH -# Copy required files to BUILD_DIR +# do the files copy +echo "[-] copying files T=$SECONDS" cd $SOURCE_DIR cp -pR --parents $(cat release/files_common) $BUILD_DIR/ cp -pR --parents $(cat $FILES_SRC) $BUILD_DIR/ -# Build + cleanup +# in the directory cd $BUILD_DIR -export PYTHONPATH="$BUILD_DIR" rm -f panda/board/obj/panda.bin.signed rm -f panda/board/obj/panda_h7.bin.signed -if [ -n "$RELEASE" ]; then - export CERT=/data/pandaextra/certs/release -fi +VERSION=$(cat common/version.h | awk -F[\"-] '{print $2}') +echo "#define COMMA_VERSION \"$VERSION-release\"" > common/version.h +echo "[-] committing version $VERSION T=$SECONDS" +git add -f . +git commit -a -m "openpilot v$VERSION release" + +# Build +export PYTHONPATH="$BUILD_DIR" scons -j$(nproc) +# release panda fw +CERT=/data/pandaextra/certs/release RELEASE=1 scons -j$(nproc) panda/ + +# Ensure no submodules in release +if test "$(git submodule--helper list | wc -l)" -gt "0"; then + echo "submodules found:" + git submodule--helper list + exit 1 +fi +git submodule status + # Cleanup find . -name '*.a' -delete find . -name '*.o' -delete @@ -45,7 +77,29 @@ find . -name '__pycache__' -delete rm -rf .sconsign.dblite Jenkinsfile release/ rm selfdrive/modeld/models/supercombo.onnx +# Restore third_party +git checkout third_party/ + # Mark as prebuilt release touch prebuilt -echo "----- openpilot has been built to $BUILD_DIR -----" +# Add built files to git +git add -f . +git commit --amend -m "openpilot v$VERSION" + +# Run tests +TEST_FILES="tools/" +cd $SOURCE_DIR +cp -pR -n --parents $TEST_FILES $BUILD_DIR/ +cd $BUILD_DIR +RELEASE=1 selfdrive/test/test_onroad.py +#selfdrive/manager/test/test_manager.py +selfdrive/car/tests/test_car_interfaces.py +rm -rf $TEST_FILES + +if [ ! -z "$RELEASE_BRANCH" ]; then + echo "[-] pushing release T=$SECONDS" + git push -f origin $RELEASE_BRANCH:$RELEASE_BRANCH +fi + +echo "[-] done T=$SECONDS" diff --git a/release/package_casync_agnos.py b/release/package_casync_agnos.py deleted file mode 100755 index b80cdfa98d..0000000000 --- a/release/package_casync_agnos.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import json -import os -import pathlib -import tempfile -import time -from openpilot.common.basedir import BASEDIR -from openpilot.system.hardware.tici.agnos import StreamingDecompressor, unsparsify, noop, AGNOS_MANIFEST_FILE -from openpilot.system.updated.casync.common import create_casync_from_file -from release.package_casync_build import upload_casync_release - - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="creates a casync release") - parser.add_argument("--manifest", type=str, help="json manifest to create agnos release from", \ - default=str(pathlib.Path(BASEDIR) / AGNOS_MANIFEST_FILE)) - args = parser.parse_args() - - manifest_file = pathlib.Path(args.manifest) - - with tempfile.TemporaryDirectory() as temp_dir: - working_dir = pathlib.Path(temp_dir) - casync_dir = working_dir / "casync" - casync_dir.mkdir() - - agnos_casync_dir = casync_dir / "agnos" - agnos_casync_dir.mkdir() - - entry_path = working_dir / "entry" - - with open(manifest_file) as f: - manifest = json.load(f) - - for entry in manifest: - print(f"creating casync agnos build from {entry}") - start = time.monotonic() - downloader = StreamingDecompressor(entry['url']) - - parse_func = unsparsify if entry['sparse'] else noop - - parsed_chunks = parse_func(downloader) - - size = entry["size"] - - cur = 0 - with open(entry_path, "wb") as f: - for chunk in parsed_chunks: - f.write(chunk) - - print(f"downloaded in {time.monotonic() - start}") - - start = time.monotonic() - agnos_filename = os.path.basename(entry["url"]).split(".")[0] - create_casync_from_file(entry_path, agnos_casync_dir, agnos_filename) - print(f"created casnc in {time.monotonic() - start}") - - upload_casync_release(casync_dir) diff --git a/release/package_casync_build.py b/release/package_casync_build.py deleted file mode 100755 index 5f92e893be..0000000000 --- a/release/package_casync_build.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python3 - -# packages a casync release, uploads to azure, and creates a manifest - -import argparse -import dataclasses -import json -import os -import pathlib -import tempfile - -from openpilot.system.hardware.tici.agnos import AGNOS_MANIFEST_FILE, get_partition_path -from openpilot.system.updated.casync.common import create_build_metadata_file, create_casync_release -from openpilot.system.version import get_build_metadata -from openpilot.tools.lib.azure_container import AzureContainer - - -BASE_URL = "https://commadist.blob.core.windows.net" - -OPENPILOT_RELEASES = f"{BASE_URL}/openpilot-releases/openpilot" -AGNOS_RELEASES = f"{BASE_URL}/openpilot-releases/agnos" - - -def create_casync_caibx(target_dir: pathlib.Path, output_dir: pathlib.Path): - output_dir.mkdir() - build_metadata = get_build_metadata() - build_metadata.openpilot.build_style = "release" if os.environ.get("RELEASE", None) is not None else "debug" - - create_build_metadata_file(target_dir, build_metadata) - - digest, caibx = create_casync_release(target_dir, output_dir, build_metadata.canonical) - - print(f"Created casync release from {target_dir} to {caibx} with digest {digest}") - - -def upload_casync_release(casync_dir: pathlib.Path): - if "AZURE_TOKEN_OPENPILOT_RELEASES" in os.environ: - os.environ["AZURE_TOKEN"] = os.environ["AZURE_TOKEN_OPENPILOT_RELEASES"] - - OPENPILOT_RELEASES_CONTAINER = AzureContainer("commadist", "openpilot-releases") - - for f in casync_dir.rglob("*"): - if f.is_file(): - blob_name = f.relative_to(casync_dir) - print(f"uploading {f} to {blob_name}") - OPENPILOT_RELEASES_CONTAINER.upload_file(str(f), str(blob_name), overwrite=True) - - -def create_partition_manifest(partition): - agnos_filename = os.path.basename(partition["url"]).split(".")[0] - - return { - "type": "partition", - "casync": { - "caibx": f"{AGNOS_RELEASES}/{agnos_filename}.caibx" - }, - "path": get_partition_path(0, partition), - "ab": True, - "size": partition["size"], - "full_check": partition["full_check"], - "hash_raw": partition["hash_raw"], - } - - -def create_openpilot_manifest(build_metadata): - return { - "type": "path_tarred", - "path": "/data/openpilot", - "casync": { - "caibx": f"{OPENPILOT_RELEASES}/{build_metadata.canonical}.caibx" - } - } - - -def create_manifest(target_dir): - with open(pathlib.Path(target_dir) / AGNOS_MANIFEST_FILE) as f: - agnos_manifest = json.load(f) - - build_metadata = get_build_metadata(args.target_dir) - - return { - "build_metadata": dataclasses.asdict(build_metadata), - "manifest": [ - *[create_partition_manifest(entry) for entry in agnos_manifest], - create_openpilot_manifest(build_metadata) - ] - } - - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="creates a casync release") - parser.add_argument("target_dir", type=str, help="path to a release build of openpilot to create release from") - args = parser.parse_args() - - target_dir = pathlib.Path(args.target_dir) - - with tempfile.TemporaryDirectory() as temp_dir: - casync_dir = pathlib.Path(temp_dir) / "casync" - casync_dir.mkdir(parents=True) - - manifest_file = pathlib.Path(temp_dir) / "manifest.json" - - create_casync_caibx(target_dir, casync_dir / "openpilot") - upload_casync_release(casync_dir) - manifest = create_manifest(target_dir) - - print(json.dumps(manifest, indent=2))