revert changes to release/ (#32364)
* revert changes to release/
* put those back
* revert Jenkinsfile
old-commit-hash: 1c42b8a05f
pull/32199/head
Adeeb Shihadeh
12 months ago
committed by
GitHub
parent
124b4566db
commit
01fcd7b727
6 changed files with 147 additions and 458 deletions
@ -1,34 +0,0 @@ |
|||||||
# openpilot releases |
|
||||||
|
|
||||||
|
|
||||||
## terms |
|
||||||
|
|
||||||
- `channel` - a named version of openpilot (git branch, casync caibx) which receives updates |
|
||||||
- `build` - a copy of openpilot ready for distribution, already built for a specific device |
|
||||||
- `build_style` - type of build, either `debug` or `release` |
|
||||||
- `debug` - build with `ALLOW_DEBUG=true`, can test experimental features like longitudinal on alpha cars |
|
||||||
- `release` - build with `ALLOW_DEBUG=false`, experimental features disabled |
|
||||||
|
|
||||||
|
|
||||||
## openpilot channels |
|
||||||
|
|
||||||
| channel | build_style | description | |
|
||||||
| ----------- | ----------- | ---------- | |
|
||||||
| release | `release` | stable release of openpilot | |
|
||||||
| staging | `release` | release candidate of openpilot for final verification | |
|
||||||
| nightly | `release` | generated nightly from last commit passing CI tests | |
|
||||||
| master | `debug` | current master commit with experimental features enabled | |
|
||||||
| git branches | `debug` | installed manually, experimental features enabled, build required | |
|
||||||
|
|
||||||
|
|
||||||
## build |
|
||||||
|
|
||||||
`release/build_release.sh <build_dir>` - creates an openpilot build into `build_dir`, ready for distribution |
|
||||||
|
|
||||||
## packaging a casync release |
|
||||||
|
|
||||||
`release/package_casync_build.py <build_dir>` - packages an openpilot build into a casync tar and uploads to `openpilot-releases` |
|
||||||
|
|
||||||
## release builds |
|
||||||
|
|
||||||
to create a release build, set `RELEASE=1` environment variable when running the build script |
|
||||||
@ -1,105 +0,0 @@ |
|||||||
#!/usr/bin/bash -e |
|
||||||
|
|
||||||
# git diff --name-status origin/release3-staging | grep "^A" | less |
|
||||||
|
|
||||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" |
|
||||||
|
|
||||||
cd $DIR |
|
||||||
|
|
||||||
BUILD_DIR=/data/openpilot |
|
||||||
SOURCE_DIR="$(git rev-parse --show-toplevel)" |
|
||||||
|
|
||||||
if [ -f /TICI ]; then |
|
||||||
FILES_SRC="release/files_tici" |
|
||||||
else |
|
||||||
echo "no release files set" |
|
||||||
exit 1 |
|
||||||
fi |
|
||||||
|
|
||||||
if [ -z "$RELEASE_BRANCH" ]; then |
|
||||||
echo "RELEASE_BRANCH is not set" |
|
||||||
exit 1 |
|
||||||
fi |
|
||||||
|
|
||||||
|
|
||||||
# set git identity |
|
||||||
source $DIR/identity.sh |
|
||||||
|
|
||||||
echo "[-] Setting up repo T=$SECONDS" |
|
||||||
rm -rf $BUILD_DIR |
|
||||||
mkdir -p $BUILD_DIR |
|
||||||
cd $BUILD_DIR |
|
||||||
git init |
|
||||||
git remote add origin git@github.com:commaai/openpilot.git |
|
||||||
git checkout --orphan $RELEASE_BRANCH |
|
||||||
|
|
||||||
# do the files copy |
|
||||||
echo "[-] copying files T=$SECONDS" |
|
||||||
cd $SOURCE_DIR |
|
||||||
cp -pR --parents $(cat release/files_common) $BUILD_DIR/ |
|
||||||
cp -pR --parents $(cat $FILES_SRC) $BUILD_DIR/ |
|
||||||
|
|
||||||
# in the directory |
|
||||||
cd $BUILD_DIR |
|
||||||
|
|
||||||
rm -f panda/board/obj/panda.bin.signed |
|
||||||
rm -f panda/board/obj/panda_h7.bin.signed |
|
||||||
|
|
||||||
VERSION=$(cat common/version.h | awk -F[\"-] '{print $2}') |
|
||||||
echo "#define COMMA_VERSION \"$VERSION-release\"" > common/version.h |
|
||||||
|
|
||||||
echo "[-] committing version $VERSION T=$SECONDS" |
|
||||||
git add -f . |
|
||||||
git commit -a -m "openpilot v$VERSION release" |
|
||||||
|
|
||||||
# Build |
|
||||||
export PYTHONPATH="$BUILD_DIR" |
|
||||||
scons -j$(nproc) |
|
||||||
|
|
||||||
# release panda fw |
|
||||||
CERT=/data/pandaextra/certs/release RELEASE=1 scons -j$(nproc) panda/ |
|
||||||
|
|
||||||
# Ensure no submodules in release |
|
||||||
if test "$(git submodule--helper list | wc -l)" -gt "0"; then |
|
||||||
echo "submodules found:" |
|
||||||
git submodule--helper list |
|
||||||
exit 1 |
|
||||||
fi |
|
||||||
git submodule status |
|
||||||
|
|
||||||
# Cleanup |
|
||||||
find . -name '*.a' -delete |
|
||||||
find . -name '*.o' -delete |
|
||||||
find . -name '*.os' -delete |
|
||||||
find . -name '*.pyc' -delete |
|
||||||
find . -name 'moc_*' -delete |
|
||||||
find . -name '__pycache__' -delete |
|
||||||
rm -rf .sconsign.dblite Jenkinsfile release/ |
|
||||||
rm selfdrive/modeld/models/supercombo.onnx |
|
||||||
|
|
||||||
# Restore third_party |
|
||||||
git checkout third_party/ |
|
||||||
|
|
||||||
# Mark as prebuilt release |
|
||||||
touch prebuilt |
|
||||||
|
|
||||||
# Add built files to git |
|
||||||
git add -f . |
|
||||||
git commit --amend -m "openpilot v$VERSION" |
|
||||||
|
|
||||||
# Run tests |
|
||||||
TEST_FILES="tools/" |
|
||||||
cd $SOURCE_DIR |
|
||||||
cp -pR -n --parents $TEST_FILES $BUILD_DIR/ |
|
||||||
cd $BUILD_DIR |
|
||||||
RELEASE=1 selfdrive/test/test_onroad.py |
|
||||||
#selfdrive/manager/test/test_manager.py |
|
||||||
#selfdrive/car/tests/test_car_interfaces.py |
|
||||||
rm -rf $TEST_FILES |
|
||||||
|
|
||||||
if [ ! -z "$RELEASE_BRANCH" ]; then |
|
||||||
echo "[-] pushing release T=$SECONDS" |
|
||||||
git push -f origin $RELEASE_BRANCH:$RELEASE_BRANCH |
|
||||||
fi |
|
||||||
|
|
||||||
echo "[-] done T=$SECONDS" |
|
||||||
@ -1,59 +0,0 @@ |
|||||||
#!/usr/bin/env python3 |
|
||||||
import argparse |
|
||||||
import json |
|
||||||
import os |
|
||||||
import pathlib |
|
||||||
import tempfile |
|
||||||
import time |
|
||||||
from openpilot.common.basedir import BASEDIR |
|
||||||
from openpilot.system.hardware.tici.agnos import StreamingDecompressor, unsparsify, noop, AGNOS_MANIFEST_FILE |
|
||||||
from openpilot.system.updated.casync.common import create_casync_from_file |
|
||||||
from release.package_casync_build import upload_casync_release |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__": |
|
||||||
parser = argparse.ArgumentParser(description="creates a casync release") |
|
||||||
parser.add_argument("--manifest", type=str, help="json manifest to create agnos release from", \ |
|
||||||
default=str(pathlib.Path(BASEDIR) / AGNOS_MANIFEST_FILE)) |
|
||||||
args = parser.parse_args() |
|
||||||
|
|
||||||
manifest_file = pathlib.Path(args.manifest) |
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory() as temp_dir: |
|
||||||
working_dir = pathlib.Path(temp_dir) |
|
||||||
casync_dir = working_dir / "casync" |
|
||||||
casync_dir.mkdir() |
|
||||||
|
|
||||||
agnos_casync_dir = casync_dir / "agnos" |
|
||||||
agnos_casync_dir.mkdir() |
|
||||||
|
|
||||||
entry_path = working_dir / "entry" |
|
||||||
|
|
||||||
with open(manifest_file) as f: |
|
||||||
manifest = json.load(f) |
|
||||||
|
|
||||||
for entry in manifest: |
|
||||||
print(f"creating casync agnos build from {entry}") |
|
||||||
start = time.monotonic() |
|
||||||
downloader = StreamingDecompressor(entry['url']) |
|
||||||
|
|
||||||
parse_func = unsparsify if entry['sparse'] else noop |
|
||||||
|
|
||||||
parsed_chunks = parse_func(downloader) |
|
||||||
|
|
||||||
size = entry["size"] |
|
||||||
|
|
||||||
cur = 0 |
|
||||||
with open(entry_path, "wb") as f: |
|
||||||
for chunk in parsed_chunks: |
|
||||||
f.write(chunk) |
|
||||||
|
|
||||||
print(f"downloaded in {time.monotonic() - start}") |
|
||||||
|
|
||||||
start = time.monotonic() |
|
||||||
agnos_filename = os.path.basename(entry["url"]).split(".")[0] |
|
||||||
create_casync_from_file(entry_path, agnos_casync_dir, agnos_filename) |
|
||||||
print(f"created casnc in {time.monotonic() - start}") |
|
||||||
|
|
||||||
upload_casync_release(casync_dir) |
|
||||||
@ -1,108 +0,0 @@ |
|||||||
#!/usr/bin/env python3 |
|
||||||
|
|
||||||
# packages a casync release, uploads to azure, and creates a manifest |
|
||||||
|
|
||||||
import argparse |
|
||||||
import dataclasses |
|
||||||
import json |
|
||||||
import os |
|
||||||
import pathlib |
|
||||||
import tempfile |
|
||||||
|
|
||||||
from openpilot.system.hardware.tici.agnos import AGNOS_MANIFEST_FILE, get_partition_path |
|
||||||
from openpilot.system.updated.casync.common import create_build_metadata_file, create_casync_release |
|
||||||
from openpilot.system.version import get_build_metadata |
|
||||||
from openpilot.tools.lib.azure_container import AzureContainer |
|
||||||
|
|
||||||
|
|
||||||
BASE_URL = "https://commadist.blob.core.windows.net" |
|
||||||
|
|
||||||
OPENPILOT_RELEASES = f"{BASE_URL}/openpilot-releases/openpilot" |
|
||||||
AGNOS_RELEASES = f"{BASE_URL}/openpilot-releases/agnos" |
|
||||||
|
|
||||||
|
|
||||||
def create_casync_caibx(target_dir: pathlib.Path, output_dir: pathlib.Path): |
|
||||||
output_dir.mkdir() |
|
||||||
build_metadata = get_build_metadata() |
|
||||||
build_metadata.openpilot.build_style = "release" if os.environ.get("RELEASE", None) is not None else "debug" |
|
||||||
|
|
||||||
create_build_metadata_file(target_dir, build_metadata) |
|
||||||
|
|
||||||
digest, caibx = create_casync_release(target_dir, output_dir, build_metadata.canonical) |
|
||||||
|
|
||||||
print(f"Created casync release from {target_dir} to {caibx} with digest {digest}") |
|
||||||
|
|
||||||
|
|
||||||
def upload_casync_release(casync_dir: pathlib.Path): |
|
||||||
if "AZURE_TOKEN_OPENPILOT_RELEASES" in os.environ: |
|
||||||
os.environ["AZURE_TOKEN"] = os.environ["AZURE_TOKEN_OPENPILOT_RELEASES"] |
|
||||||
|
|
||||||
OPENPILOT_RELEASES_CONTAINER = AzureContainer("commadist", "openpilot-releases") |
|
||||||
|
|
||||||
for f in casync_dir.rglob("*"): |
|
||||||
if f.is_file(): |
|
||||||
blob_name = f.relative_to(casync_dir) |
|
||||||
print(f"uploading {f} to {blob_name}") |
|
||||||
OPENPILOT_RELEASES_CONTAINER.upload_file(str(f), str(blob_name), overwrite=True) |
|
||||||
|
|
||||||
|
|
||||||
def create_partition_manifest(partition): |
|
||||||
agnos_filename = os.path.basename(partition["url"]).split(".")[0] |
|
||||||
|
|
||||||
return { |
|
||||||
"type": "partition", |
|
||||||
"casync": { |
|
||||||
"caibx": f"{AGNOS_RELEASES}/{agnos_filename}.caibx" |
|
||||||
}, |
|
||||||
"path": get_partition_path(0, partition), |
|
||||||
"ab": True, |
|
||||||
"size": partition["size"], |
|
||||||
"full_check": partition["full_check"], |
|
||||||
"hash_raw": partition["hash_raw"], |
|
||||||
} |
|
||||||
|
|
||||||
|
|
||||||
def create_openpilot_manifest(build_metadata): |
|
||||||
return { |
|
||||||
"type": "path_tarred", |
|
||||||
"path": "/data/openpilot", |
|
||||||
"casync": { |
|
||||||
"caibx": f"{OPENPILOT_RELEASES}/{build_metadata.canonical}.caibx" |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
|
|
||||||
def create_manifest(target_dir): |
|
||||||
with open(pathlib.Path(target_dir) / AGNOS_MANIFEST_FILE) as f: |
|
||||||
agnos_manifest = json.load(f) |
|
||||||
|
|
||||||
build_metadata = get_build_metadata(args.target_dir) |
|
||||||
|
|
||||||
return { |
|
||||||
"build_metadata": dataclasses.asdict(build_metadata), |
|
||||||
"manifest": [ |
|
||||||
*[create_partition_manifest(entry) for entry in agnos_manifest], |
|
||||||
create_openpilot_manifest(build_metadata) |
|
||||||
] |
|
||||||
} |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__": |
|
||||||
parser = argparse.ArgumentParser(description="creates a casync release") |
|
||||||
parser.add_argument("target_dir", type=str, help="path to a release build of openpilot to create release from") |
|
||||||
args = parser.parse_args() |
|
||||||
|
|
||||||
target_dir = pathlib.Path(args.target_dir) |
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory() as temp_dir: |
|
||||||
casync_dir = pathlib.Path(temp_dir) / "casync" |
|
||||||
casync_dir.mkdir(parents=True) |
|
||||||
|
|
||||||
manifest_file = pathlib.Path(temp_dir) / "manifest.json" |
|
||||||
|
|
||||||
create_casync_caibx(target_dir, casync_dir / "openpilot") |
|
||||||
upload_casync_release(casync_dir) |
|
||||||
manifest = create_manifest(target_dir) |
|
||||||
|
|
||||||
print(json.dumps(manifest, indent=2)) |
|
||||||
Loading…
Reference in new issue