revert changes to release/ (#32364)
* revert changes to release/
* put those back
* revert Jenkinsfile
old-commit-hash: 1c42b8a05f
pull/32199/head
Adeeb Shihadeh
12 months ago
committed by
GitHub
parent
124b4566db
commit
01fcd7b727
6 changed files with 147 additions and 458 deletions
@ -1,34 +0,0 @@ |
||||
# openpilot releases |
||||
|
||||
|
||||
## terms |
||||
|
||||
- `channel` - a named version of openpilot (git branch, casync caibx) which receives updates |
||||
- `build` - a copy of openpilot ready for distribution, already built for a specific device |
||||
- `build_style` - type of build, either `debug` or `release` |
||||
- `debug` - build with `ALLOW_DEBUG=true`, can test experimental features like longitudinal on alpha cars |
||||
- `release` - build with `ALLOW_DEBUG=false`, experimental features disabled |
||||
|
||||
|
||||
## openpilot channels |
||||
|
||||
| channel | build_style | description | |
||||
| ----------- | ----------- | ---------- | |
||||
| release | `release` | stable release of openpilot | |
||||
| staging | `release` | release candidate of openpilot for final verification | |
||||
| nightly | `release` | generated nightly from last commit passing CI tests | |
||||
| master | `debug` | current master commit with experimental features enabled | |
||||
| git branches | `debug` | installed manually, experimental features enabled, build required | |
||||
|
||||
|
||||
## build |
||||
|
||||
`release/build_release.sh <build_dir>` - creates an openpilot build into `build_dir`, ready for distribution |
||||
|
||||
## packaging a casync release |
||||
|
||||
`release/package_casync_build.py <build_dir>` - packages an openpilot build into a casync tar and uploads to `openpilot-releases` |
||||
|
||||
## release builds |
||||
|
||||
to create a release build, set `RELEASE=1` environment variable when running the build script |
||||
@ -1,105 +0,0 @@ |
||||
#!/usr/bin/bash -e |
||||
|
||||
# git diff --name-status origin/release3-staging | grep "^A" | less |
||||
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" |
||||
|
||||
cd $DIR |
||||
|
||||
BUILD_DIR=/data/openpilot |
||||
SOURCE_DIR="$(git rev-parse --show-toplevel)" |
||||
|
||||
if [ -f /TICI ]; then |
||||
FILES_SRC="release/files_tici" |
||||
else |
||||
echo "no release files set" |
||||
exit 1 |
||||
fi |
||||
|
||||
if [ -z "$RELEASE_BRANCH" ]; then |
||||
echo "RELEASE_BRANCH is not set" |
||||
exit 1 |
||||
fi |
||||
|
||||
|
||||
# set git identity |
||||
source $DIR/identity.sh |
||||
|
||||
echo "[-] Setting up repo T=$SECONDS" |
||||
rm -rf $BUILD_DIR |
||||
mkdir -p $BUILD_DIR |
||||
cd $BUILD_DIR |
||||
git init |
||||
git remote add origin git@github.com:commaai/openpilot.git |
||||
git checkout --orphan $RELEASE_BRANCH |
||||
|
||||
# do the files copy |
||||
echo "[-] copying files T=$SECONDS" |
||||
cd $SOURCE_DIR |
||||
cp -pR --parents $(cat release/files_common) $BUILD_DIR/ |
||||
cp -pR --parents $(cat $FILES_SRC) $BUILD_DIR/ |
||||
|
||||
# in the directory |
||||
cd $BUILD_DIR |
||||
|
||||
rm -f panda/board/obj/panda.bin.signed |
||||
rm -f panda/board/obj/panda_h7.bin.signed |
||||
|
||||
VERSION=$(cat common/version.h | awk -F[\"-] '{print $2}') |
||||
echo "#define COMMA_VERSION \"$VERSION-release\"" > common/version.h |
||||
|
||||
echo "[-] committing version $VERSION T=$SECONDS" |
||||
git add -f . |
||||
git commit -a -m "openpilot v$VERSION release" |
||||
|
||||
# Build |
||||
export PYTHONPATH="$BUILD_DIR" |
||||
scons -j$(nproc) |
||||
|
||||
# release panda fw |
||||
CERT=/data/pandaextra/certs/release RELEASE=1 scons -j$(nproc) panda/ |
||||
|
||||
# Ensure no submodules in release |
||||
if test "$(git submodule--helper list | wc -l)" -gt "0"; then |
||||
echo "submodules found:" |
||||
git submodule--helper list |
||||
exit 1 |
||||
fi |
||||
git submodule status |
||||
|
||||
# Cleanup |
||||
find . -name '*.a' -delete |
||||
find . -name '*.o' -delete |
||||
find . -name '*.os' -delete |
||||
find . -name '*.pyc' -delete |
||||
find . -name 'moc_*' -delete |
||||
find . -name '__pycache__' -delete |
||||
rm -rf .sconsign.dblite Jenkinsfile release/ |
||||
rm selfdrive/modeld/models/supercombo.onnx |
||||
|
||||
# Restore third_party |
||||
git checkout third_party/ |
||||
|
||||
# Mark as prebuilt release |
||||
touch prebuilt |
||||
|
||||
# Add built files to git |
||||
git add -f . |
||||
git commit --amend -m "openpilot v$VERSION" |
||||
|
||||
# Run tests |
||||
TEST_FILES="tools/" |
||||
cd $SOURCE_DIR |
||||
cp -pR -n --parents $TEST_FILES $BUILD_DIR/ |
||||
cd $BUILD_DIR |
||||
RELEASE=1 selfdrive/test/test_onroad.py |
||||
#selfdrive/manager/test/test_manager.py |
||||
#selfdrive/car/tests/test_car_interfaces.py |
||||
rm -rf $TEST_FILES |
||||
|
||||
if [ ! -z "$RELEASE_BRANCH" ]; then |
||||
echo "[-] pushing release T=$SECONDS" |
||||
git push -f origin $RELEASE_BRANCH:$RELEASE_BRANCH |
||||
fi |
||||
|
||||
echo "[-] done T=$SECONDS" |
||||
@ -1,59 +0,0 @@ |
||||
#!/usr/bin/env python3 |
||||
import argparse |
||||
import json |
||||
import os |
||||
import pathlib |
||||
import tempfile |
||||
import time |
||||
from openpilot.common.basedir import BASEDIR |
||||
from openpilot.system.hardware.tici.agnos import StreamingDecompressor, unsparsify, noop, AGNOS_MANIFEST_FILE |
||||
from openpilot.system.updated.casync.common import create_casync_from_file |
||||
from release.package_casync_build import upload_casync_release |
||||
|
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
parser = argparse.ArgumentParser(description="creates a casync release") |
||||
parser.add_argument("--manifest", type=str, help="json manifest to create agnos release from", \ |
||||
default=str(pathlib.Path(BASEDIR) / AGNOS_MANIFEST_FILE)) |
||||
args = parser.parse_args() |
||||
|
||||
manifest_file = pathlib.Path(args.manifest) |
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir: |
||||
working_dir = pathlib.Path(temp_dir) |
||||
casync_dir = working_dir / "casync" |
||||
casync_dir.mkdir() |
||||
|
||||
agnos_casync_dir = casync_dir / "agnos" |
||||
agnos_casync_dir.mkdir() |
||||
|
||||
entry_path = working_dir / "entry" |
||||
|
||||
with open(manifest_file) as f: |
||||
manifest = json.load(f) |
||||
|
||||
for entry in manifest: |
||||
print(f"creating casync agnos build from {entry}") |
||||
start = time.monotonic() |
||||
downloader = StreamingDecompressor(entry['url']) |
||||
|
||||
parse_func = unsparsify if entry['sparse'] else noop |
||||
|
||||
parsed_chunks = parse_func(downloader) |
||||
|
||||
size = entry["size"] |
||||
|
||||
cur = 0 |
||||
with open(entry_path, "wb") as f: |
||||
for chunk in parsed_chunks: |
||||
f.write(chunk) |
||||
|
||||
print(f"downloaded in {time.monotonic() - start}") |
||||
|
||||
start = time.monotonic() |
||||
agnos_filename = os.path.basename(entry["url"]).split(".")[0] |
||||
create_casync_from_file(entry_path, agnos_casync_dir, agnos_filename) |
||||
print(f"created casnc in {time.monotonic() - start}") |
||||
|
||||
upload_casync_release(casync_dir) |
||||
@ -1,108 +0,0 @@ |
||||
#!/usr/bin/env python3 |
||||
|
||||
# packages a casync release, uploads to azure, and creates a manifest |
||||
|
||||
import argparse |
||||
import dataclasses |
||||
import json |
||||
import os |
||||
import pathlib |
||||
import tempfile |
||||
|
||||
from openpilot.system.hardware.tici.agnos import AGNOS_MANIFEST_FILE, get_partition_path |
||||
from openpilot.system.updated.casync.common import create_build_metadata_file, create_casync_release |
||||
from openpilot.system.version import get_build_metadata |
||||
from openpilot.tools.lib.azure_container import AzureContainer |
||||
|
||||
|
||||
BASE_URL = "https://commadist.blob.core.windows.net" |
||||
|
||||
OPENPILOT_RELEASES = f"{BASE_URL}/openpilot-releases/openpilot" |
||||
AGNOS_RELEASES = f"{BASE_URL}/openpilot-releases/agnos" |
||||
|
||||
|
||||
def create_casync_caibx(target_dir: pathlib.Path, output_dir: pathlib.Path): |
||||
output_dir.mkdir() |
||||
build_metadata = get_build_metadata() |
||||
build_metadata.openpilot.build_style = "release" if os.environ.get("RELEASE", None) is not None else "debug" |
||||
|
||||
create_build_metadata_file(target_dir, build_metadata) |
||||
|
||||
digest, caibx = create_casync_release(target_dir, output_dir, build_metadata.canonical) |
||||
|
||||
print(f"Created casync release from {target_dir} to {caibx} with digest {digest}") |
||||
|
||||
|
||||
def upload_casync_release(casync_dir: pathlib.Path): |
||||
if "AZURE_TOKEN_OPENPILOT_RELEASES" in os.environ: |
||||
os.environ["AZURE_TOKEN"] = os.environ["AZURE_TOKEN_OPENPILOT_RELEASES"] |
||||
|
||||
OPENPILOT_RELEASES_CONTAINER = AzureContainer("commadist", "openpilot-releases") |
||||
|
||||
for f in casync_dir.rglob("*"): |
||||
if f.is_file(): |
||||
blob_name = f.relative_to(casync_dir) |
||||
print(f"uploading {f} to {blob_name}") |
||||
OPENPILOT_RELEASES_CONTAINER.upload_file(str(f), str(blob_name), overwrite=True) |
||||
|
||||
|
||||
def create_partition_manifest(partition): |
||||
agnos_filename = os.path.basename(partition["url"]).split(".")[0] |
||||
|
||||
return { |
||||
"type": "partition", |
||||
"casync": { |
||||
"caibx": f"{AGNOS_RELEASES}/{agnos_filename}.caibx" |
||||
}, |
||||
"path": get_partition_path(0, partition), |
||||
"ab": True, |
||||
"size": partition["size"], |
||||
"full_check": partition["full_check"], |
||||
"hash_raw": partition["hash_raw"], |
||||
} |
||||
|
||||
|
||||
def create_openpilot_manifest(build_metadata): |
||||
return { |
||||
"type": "path_tarred", |
||||
"path": "/data/openpilot", |
||||
"casync": { |
||||
"caibx": f"{OPENPILOT_RELEASES}/{build_metadata.canonical}.caibx" |
||||
} |
||||
} |
||||
|
||||
|
||||
def create_manifest(target_dir): |
||||
with open(pathlib.Path(target_dir) / AGNOS_MANIFEST_FILE) as f: |
||||
agnos_manifest = json.load(f) |
||||
|
||||
build_metadata = get_build_metadata(args.target_dir) |
||||
|
||||
return { |
||||
"build_metadata": dataclasses.asdict(build_metadata), |
||||
"manifest": [ |
||||
*[create_partition_manifest(entry) for entry in agnos_manifest], |
||||
create_openpilot_manifest(build_metadata) |
||||
] |
||||
} |
||||
|
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
parser = argparse.ArgumentParser(description="creates a casync release") |
||||
parser.add_argument("target_dir", type=str, help="path to a release build of openpilot to create release from") |
||||
args = parser.parse_args() |
||||
|
||||
target_dir = pathlib.Path(args.target_dir) |
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir: |
||||
casync_dir = pathlib.Path(temp_dir) / "casync" |
||||
casync_dir.mkdir(parents=True) |
||||
|
||||
manifest_file = pathlib.Path(temp_dir) / "manifest.json" |
||||
|
||||
create_casync_caibx(target_dir, casync_dir / "openpilot") |
||||
upload_casync_release(casync_dir) |
||||
manifest = create_manifest(target_dir) |
||||
|
||||
print(json.dumps(manifest, indent=2)) |
||||
Loading…
Reference in new issue