diff --git a/Jenkinsfile b/Jenkinsfile index 4afb3964f7..efe598c275 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -142,6 +142,23 @@ def setupCredentials() { } +def build_release(String channel_name) { + return parallel ( + "${channel_name} (git)": { + deviceStage("build git", "tici-needs-can", [], [ + ["build ${channel_name}", "RELEASE_BRANCH=${channel_name} $SOURCE_DIR/release/build_release.sh"], + ]) + }, + "${channel_name} (casync)": { + deviceStage("build casync", "tici-needs-can", [], [ + ["build ${channel_name}", "RELEASE=1 OPENPILOT_CHANNEL=${channel_name} BUILD_DIR=/data/openpilot_build CASYNC_DIR=/data/casync $SOURCE_DIR/release/create_casync_build.sh"], + //["upload ${channel_name}", "OPENPILOT_CHANNEL=${channel_name} $SOURCE_DIR/release/upload_casync_release.sh"], + ]) + } + ) +} + + node { env.CI = "1" env.PYTHONWARNINGS = "error" @@ -164,15 +181,11 @@ node { try { if (env.BRANCH_NAME == 'devel-staging') { - deviceStage("build release3-staging", "tici-needs-can", [], [ - ["build release3-staging", "RELEASE_BRANCH=release3-staging $SOURCE_DIR/release/build_release.sh"], - ]) + build_release("release3-staging") } if (env.BRANCH_NAME == 'master-ci') { - deviceStage("build nightly", "tici-needs-can", [], [ - ["build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"], - ]) + build_release("nightly") } if (!env.BRANCH_NAME.matches(excludeRegex)) { diff --git a/release/README.md b/release/README.md new file mode 100644 index 0000000000..77cd15ad69 --- /dev/null +++ b/release/README.md @@ -0,0 +1,44 @@ +# openpilot releases + + +## terms + +- `channel` - a named version of openpilot (git branch, casync caidx) which receives updates +- `build` - a release which is already built for the comma 3/3x and contains only required files for running openpilot and identifying the release + +- `build_style` - type of build, either `debug` or `release` + - `debug` - build with `ALLOW_DEBUG=true`, can test experimental features like longitudinal on alpha cars + - `release` - build with `ALLOW_DEBUG=false`, experimental features disabled + + +## openpilot channels + +| channel | build_style | description | +| ----------- | ----------- | ---------- | +| release | `release` | stable release of openpilot | +| staging | `release` | release candidate of openpilot for final verification | +| nightly | `release` | generated nightly from last commit passing CI tests | +| master | `debug` | current master commit with experimental features enabled | +| git branches | `debug` | installed manually, experimental features enabled, build required | + + +## creating casync build + +`create_casync_build.sh` - creates a casync openpilot build, ready to upload to `openpilot-releases` + +```bash +# run on a tici, within the directory you want to create the build from. +# creates a prebuilt version of openpilot into BUILD_DIR and outputs the caidx +# and other casync files into CASYNC_DIR for uploading to openpilot-releases. +BUILD_DIR=/data/openpilot_build \ +CASYNC_DIR=/data/casync \ +OPENPILOT_CHANNEL=nightly \ +release/create_casync_build.sh +``` + +`upload_casync_release.sh` - helper for uploading a casync build to `openpilot-releases` + + +## release builds + +to create a release build, set `RELEASE=1` environment variable when running the build script diff --git a/release/copy_build_files.sh b/release/copy_build_files.sh new file mode 100755 index 0000000000..31fd8af778 --- /dev/null +++ b/release/copy_build_files.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +SOURCE_DIR=$1 +TARGET_DIR=$2 + +if [ -f /TICI ]; then + FILES_SRC="release/files_tici" +else + echo "no release files set" + exit 1 +fi + +cd $SOURCE_DIR +cp -pR --parents $(cat release/files_common) $BUILD_DIR/ +cp -pR --parents $(cat $FILES_SRC) $TARGET_DIR/ diff --git a/release/create_casync_build.sh b/release/create_casync_build.sh new file mode 100755 index 0000000000..d41909a6ab --- /dev/null +++ b/release/create_casync_build.sh @@ -0,0 +1,21 @@ +#!/usr/bin/bash + +set -ex + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" + +CASYNC_DIR="${CASYNC_DIR:=/tmp/casync}" +SOURCE_DIR="$(git -C $DIR rev-parse --show-toplevel)" +BUILD_DIR="${BUILD_DIR:=$(mktemp -d)}" + +echo "Creating casync release from $SOURCE_DIR to $CASYNC_DIR" + +mkdir -p $CASYNC_DIR +rm -rf $BUILD_DIR +mkdir -p $BUILD_DIR + +release/copy_build_files.sh $SOURCE_DIR $BUILD_DIR +release/create_prebuilt.sh $BUILD_DIR + +cd $SOURCE_DIR +release/create_casync_release.py $BUILD_DIR $CASYNC_DIR $OPENPILOT_CHANNEL diff --git a/release/create_casync_release.py b/release/create_casync_release.py new file mode 100755 index 0000000000..8977e09abc --- /dev/null +++ b/release/create_casync_release.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python + +import argparse +import pathlib + +from openpilot.system.updated.casync.common import create_caexclude_file, create_casync_release, create_build_metadata_file +from openpilot.system.version import get_build_metadata + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="creates a casync release") + parser.add_argument("target_dir", type=str, help="target directory to build channel from") + parser.add_argument("output_dir", type=str, help="output directory for the channel") + parser.add_argument("channel", type=str, help="what channel this build is") + args = parser.parse_args() + + target_dir = pathlib.Path(args.target_dir) + output_dir = pathlib.Path(args.output_dir) + + create_build_metadata_file(target_dir, get_build_metadata(), args.channel) + create_caexclude_file(target_dir) + + digest, caidx = create_casync_release(target_dir, output_dir, args.channel) + + print(f"Created casync release from {target_dir} to {caidx} with digest {digest}") diff --git a/release/create_prebuilt.sh b/release/create_prebuilt.sh new file mode 100755 index 0000000000..6d3768c536 --- /dev/null +++ b/release/create_prebuilt.sh @@ -0,0 +1,34 @@ +#!/usr/bin/bash -e + +# runs on tici to create a prebuilt version of a release + +set -ex + +BUILD_DIR=$1 + +cd $BUILD_DIR + +# Build +export PYTHONPATH="$BUILD_DIR" + +rm -f panda/board/obj/panda.bin.signed +rm -f panda/board/obj/panda_h7.bin.signed + +if [ -n "$RELEASE" ]; then + export CERT=/data/pandaextra/certs/release +fi + +scons -j$(nproc) + +# Cleanup +find . -name '*.a' -delete +find . -name '*.o' -delete +find . -name '*.os' -delete +find . -name '*.pyc' -delete +find . -name 'moc_*' -delete +find . -name '__pycache__' -delete +rm -rf .sconsign.dblite Jenkinsfile release/ +rm selfdrive/modeld/models/supercombo.onnx + +# Mark as prebuilt release +touch prebuilt diff --git a/release/upload_casync_release.sh b/release/upload_casync_release.sh new file mode 100755 index 0000000000..02ced8338e --- /dev/null +++ b/release/upload_casync_release.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +CASYNC_DIR="${CASYNC_DIR:=/tmp/casync}" + +OPENPILOT_RELEASES="https://commadist.blob.core.windows.net/openpilot-releases/" + +SAS="$(python -c 'from tools.lib.azure_container import get_container_sas;print(get_container_sas("commadist","openpilot-releases"))')" + +azcopy cp "$CASYNC_DIR*" "$OPENPILOT_RELEASES?$SAS" --recursive diff --git a/system/updated/casync/common.py b/system/updated/casync/common.py new file mode 100644 index 0000000000..1703c74a76 --- /dev/null +++ b/system/updated/casync/common.py @@ -0,0 +1,53 @@ +import dataclasses +import json +import pathlib +import subprocess + +from openpilot.system.version import BUILD_METADATA_FILENAME, BuildMetadata + + +CASYNC_ARGS = ["--with=symlinks", "--with=permissions"] +CASYNC_FILES = [BUILD_METADATA_FILENAME, ".caexclude"] + + +def run(cmd): + return subprocess.check_output(cmd) + + +def get_exclude_set(path) -> set[str]: + exclude_set = set(CASYNC_FILES) + + for file in path.rglob("*"): + if file.is_file() or file.is_symlink(): + + while file.resolve() != path.resolve(): + exclude_set.add(str(file.relative_to(path))) + + file = file.parent + + return exclude_set + + +def create_caexclude_file(path: pathlib.Path): + with open(path / ".caexclude", "w") as f: + # exclude everything except the paths already in the release + f.write("*\n") + f.write(".*\n") + + for file in sorted(get_exclude_set(path)): + f.write(f"!{file}\n") + + +def create_build_metadata_file(path: pathlib.Path, build_metadata: BuildMetadata, channel: str): + with open(path / BUILD_METADATA_FILENAME, "w") as f: + build_metadata_dict = dataclasses.asdict(build_metadata) + build_metadata_dict["channel"] = channel + build_metadata_dict["openpilot"].pop("is_dirty") # this is determined at runtime + f.write(json.dumps(build_metadata_dict)) + + +def create_casync_release(target_dir: pathlib.Path, output_dir: pathlib.Path, channel: str): + caidx_file = output_dir / f"{channel}.caidx" + run(["casync", "make", *CASYNC_ARGS, caidx_file, target_dir]) + digest = run(["casync", "digest", *CASYNC_ARGS, target_dir]).decode("utf-8").strip() + return digest, caidx_file