consolidate casync build into fewer scripts and fix pc release build (#32225)
* less scripts * better * fixes * naming * revert * cleanup * lets test it * fix that one * and rm * don't run this * fix * not here * revert testing * fix docs * default here too * t --------- Co-authored-by: Comma Device <device@comma.ai>pull/32245/head
Justin Newberry
1 year ago
committed by
GitHub
parent
746901e6b0
commit
62f053bc4d
12 changed files with 187 additions and 219 deletions
@ -0,0 +1,51 @@ |
||||
#!/usr/bin/bash |
||||
|
||||
set -e |
||||
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" |
||||
SOURCE_DIR="$(git -C $DIR rev-parse --show-toplevel)" |
||||
BUILD_DIR=${1:-$(mktemp -d)} |
||||
|
||||
if [ -f /TICI ]; then |
||||
FILES_SRC="release/files_tici" |
||||
else |
||||
FILES_SRC="release/files_pc" |
||||
fi |
||||
|
||||
echo "Building openpilot into $BUILD_DIR" |
||||
|
||||
rm -rf $BUILD_DIR |
||||
mkdir -p $BUILD_DIR |
||||
|
||||
# Copy required files to BUILD_DIR |
||||
cd $SOURCE_DIR |
||||
cp -pR --parents $(cat release/files_common) $BUILD_DIR/ |
||||
cp -pR --parents $(cat $FILES_SRC) $BUILD_DIR/ |
||||
|
||||
# Build + cleanup |
||||
cd $BUILD_DIR |
||||
export PYTHONPATH="$BUILD_DIR" |
||||
|
||||
rm -f panda/board/obj/panda.bin.signed |
||||
rm -f panda/board/obj/panda_h7.bin.signed |
||||
|
||||
if [ -n "$RELEASE" ]; then |
||||
export CERT=/data/pandaextra/certs/release |
||||
fi |
||||
|
||||
scons -j$(nproc) |
||||
|
||||
# Cleanup |
||||
find . -name '*.a' -delete |
||||
find . -name '*.o' -delete |
||||
find . -name '*.os' -delete |
||||
find . -name '*.pyc' -delete |
||||
find . -name 'moc_*' -delete |
||||
find . -name '__pycache__' -delete |
||||
rm -rf .sconsign.dblite Jenkinsfile release/ |
||||
rm selfdrive/modeld/models/supercombo.onnx |
||||
|
||||
# Mark as prebuilt release |
||||
touch prebuilt |
||||
|
||||
echo "----- openpilot has been built to $BUILD_DIR -----" |
||||
@ -1,15 +0,0 @@ |
||||
#!/bin/bash |
||||
|
||||
SOURCE_DIR=$1 |
||||
TARGET_DIR=$2 |
||||
|
||||
if [ -f /TICI ]; then |
||||
FILES_SRC="release/files_tici" |
||||
else |
||||
echo "no release files set" |
||||
exit 1 |
||||
fi |
||||
|
||||
cd $SOURCE_DIR |
||||
cp -pR --parents $(cat release/files_common) $TARGET_DIR/ |
||||
cp -pR --parents $(cat $FILES_SRC) $TARGET_DIR/ |
||||
@ -1,23 +0,0 @@ |
||||
#!/usr/bin/bash |
||||
|
||||
set -ex |
||||
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" |
||||
|
||||
CASYNC_DIR="${CASYNC_DIR:=/tmp/casync}" |
||||
SOURCE_DIR="$(git -C $DIR rev-parse --show-toplevel)" |
||||
BUILD_DIR="${BUILD_DIR:=$(mktemp -d)}" |
||||
PYTHONPATH="$SOURCE_DIR" |
||||
|
||||
echo "Creating casync release from $SOURCE_DIR to $CASYNC_DIR" |
||||
|
||||
cd $SOURCE_DIR |
||||
mkdir -p $CASYNC_DIR |
||||
rm -rf $BUILD_DIR |
||||
mkdir -p $BUILD_DIR |
||||
|
||||
release/copy_build_files.sh $SOURCE_DIR $BUILD_DIR |
||||
release/create_prebuilt.sh $BUILD_DIR |
||||
|
||||
cd $SOURCE_DIR |
||||
release/create_casync_release.py $BUILD_DIR $CASYNC_DIR |
||||
@ -1,27 +0,0 @@ |
||||
#!/usr/bin/env python |
||||
|
||||
import argparse |
||||
import os |
||||
import pathlib |
||||
|
||||
from openpilot.system.updated.casync.common import create_casync_release, create_build_metadata_file |
||||
from openpilot.system.version import get_build_metadata |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
parser = argparse.ArgumentParser(description="creates a casync release") |
||||
parser.add_argument("target_dir", type=str, help="target directory to build channel from") |
||||
parser.add_argument("output_dir", type=str, help="output directory for the channel") |
||||
args = parser.parse_args() |
||||
|
||||
target_dir = pathlib.Path(args.target_dir) |
||||
output_dir = pathlib.Path(args.output_dir) |
||||
|
||||
build_metadata = get_build_metadata() |
||||
build_metadata.openpilot.build_style = "release" if os.environ.get("RELEASE", None) is not None else "debug" |
||||
|
||||
create_build_metadata_file(target_dir, build_metadata) |
||||
|
||||
digest, caibx = create_casync_release(target_dir, output_dir, build_metadata.canonical) |
||||
|
||||
print(f"Created casync release from {target_dir} to {caibx} with digest {digest}") |
||||
@ -1,34 +0,0 @@ |
||||
#!/usr/bin/bash -e |
||||
|
||||
# runs on tici to create a prebuilt version of a release |
||||
|
||||
set -ex |
||||
|
||||
BUILD_DIR=$1 |
||||
|
||||
cd $BUILD_DIR |
||||
|
||||
# Build |
||||
export PYTHONPATH="$BUILD_DIR" |
||||
|
||||
rm -f panda/board/obj/panda.bin.signed |
||||
rm -f panda/board/obj/panda_h7.bin.signed |
||||
|
||||
if [ -n "$RELEASE" ]; then |
||||
export CERT=/data/pandaextra/certs/release |
||||
fi |
||||
|
||||
scons -j$(nproc) |
||||
|
||||
# Cleanup |
||||
find . -name '*.a' -delete |
||||
find . -name '*.o' -delete |
||||
find . -name '*.os' -delete |
||||
find . -name '*.pyc' -delete |
||||
find . -name 'moc_*' -delete |
||||
find . -name '__pycache__' -delete |
||||
rm -rf .sconsign.dblite Jenkinsfile release/ |
||||
rm selfdrive/modeld/models/supercombo.onnx |
||||
|
||||
# Mark as prebuilt release |
||||
touch prebuilt |
||||
@ -1,64 +0,0 @@ |
||||
#!/usr/bin/env python3 |
||||
import argparse |
||||
import dataclasses |
||||
import json |
||||
import os |
||||
import pathlib |
||||
|
||||
from openpilot.system.hardware.tici.agnos import AGNOS_MANIFEST_FILE, get_partition_path |
||||
from openpilot.system.version import get_build_metadata |
||||
|
||||
|
||||
BASE_URL = "https://commadist.blob.core.windows.net" |
||||
|
||||
OPENPILOT_RELEASES = f"{BASE_URL}/openpilot-releases/openpilot" |
||||
AGNOS_RELEASES = f"{BASE_URL}/openpilot-releases/agnos" |
||||
|
||||
|
||||
def create_partition_manifest(partition): |
||||
agnos_filename = os.path.basename(partition["url"]).split(".")[0] |
||||
|
||||
return { |
||||
"type": "partition", |
||||
"casync": { |
||||
"caibx": f"{AGNOS_RELEASES}/{agnos_filename}.caibx" |
||||
}, |
||||
"path": get_partition_path(0, partition), |
||||
"ab": True, |
||||
"size": partition["size"], |
||||
"full_check": partition["full_check"], |
||||
"hash_raw": partition["hash_raw"], |
||||
} |
||||
|
||||
|
||||
def create_openpilot_manifest(build_metadata): |
||||
return { |
||||
"type": "path_tarred", |
||||
"path": "/data/openpilot", |
||||
"casync": { |
||||
"caibx": f"{OPENPILOT_RELEASES}/{build_metadata.canonical}.caibx" |
||||
} |
||||
} |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
parser = argparse.ArgumentParser(description="creates a casync release") |
||||
parser.add_argument("target_dir", type=str, help="directory of the channel to create manifest from") |
||||
parser.add_argument("output_file", type=str, help="output file to put the manifest") |
||||
args = parser.parse_args() |
||||
|
||||
with open(pathlib.Path(args.target_dir) / AGNOS_MANIFEST_FILE) as f: |
||||
agnos_manifest = json.load(f) |
||||
|
||||
build_metadata = get_build_metadata(args.target_dir) |
||||
|
||||
ret = { |
||||
"build_metadata": dataclasses.asdict(build_metadata), |
||||
"manifest": [ |
||||
*[create_partition_manifest(entry) for entry in agnos_manifest], |
||||
create_openpilot_manifest(build_metadata) |
||||
] |
||||
} |
||||
|
||||
with open(args.output_file, "w") as f: |
||||
f.write(json.dumps(ret, indent=2)) |
||||
@ -0,0 +1,108 @@ |
||||
#!/usr/bin/env python3 |
||||
|
||||
# packages a casync release, uploads to azure, and creates a manifest |
||||
|
||||
import argparse |
||||
import dataclasses |
||||
import json |
||||
import os |
||||
import pathlib |
||||
import tempfile |
||||
|
||||
from openpilot.system.hardware.tici.agnos import AGNOS_MANIFEST_FILE, get_partition_path |
||||
from openpilot.system.updated.casync.common import create_build_metadata_file, create_casync_release |
||||
from openpilot.system.version import get_build_metadata |
||||
from openpilot.tools.lib.azure_container import AzureContainer |
||||
|
||||
|
||||
BASE_URL = "https://commadist.blob.core.windows.net" |
||||
|
||||
OPENPILOT_RELEASES = f"{BASE_URL}/openpilot-releases/openpilot" |
||||
AGNOS_RELEASES = f"{BASE_URL}/openpilot-releases/agnos" |
||||
|
||||
|
||||
def create_casync_caibx(target_dir: pathlib.Path, output_dir: pathlib.Path): |
||||
output_dir.mkdir() |
||||
build_metadata = get_build_metadata() |
||||
build_metadata.openpilot.build_style = "release" if os.environ.get("RELEASE", None) is not None else "debug" |
||||
|
||||
create_build_metadata_file(target_dir, build_metadata) |
||||
|
||||
digest, caibx = create_casync_release(target_dir, output_dir, build_metadata.canonical) |
||||
|
||||
print(f"Created casync release from {target_dir} to {caibx} with digest {digest}") |
||||
|
||||
|
||||
def upload_casync_release(casync_dir: pathlib.Path): |
||||
if "AZURE_TOKEN_OPENPILOT_RELEASES" in os.environ: |
||||
os.environ["AZURE_TOKEN"] = os.environ["AZURE_TOKEN_OPENPILOT_RELEASES"] |
||||
|
||||
OPENPILOT_RELEASES_CONTAINER = AzureContainer("commadist", "openpilot-releases") |
||||
|
||||
for f in casync_dir.rglob("*"): |
||||
if f.is_file(): |
||||
blob_name = f.relative_to(casync_dir) |
||||
print(f"uploading {f} to {blob_name}") |
||||
OPENPILOT_RELEASES_CONTAINER.upload_file(str(f), str(blob_name), overwrite=True) |
||||
|
||||
|
||||
def create_partition_manifest(partition): |
||||
agnos_filename = os.path.basename(partition["url"]).split(".")[0] |
||||
|
||||
return { |
||||
"type": "partition", |
||||
"casync": { |
||||
"caibx": f"{AGNOS_RELEASES}/{agnos_filename}.caibx" |
||||
}, |
||||
"path": get_partition_path(0, partition), |
||||
"ab": True, |
||||
"size": partition["size"], |
||||
"full_check": partition["full_check"], |
||||
"hash_raw": partition["hash_raw"], |
||||
} |
||||
|
||||
|
||||
def create_openpilot_manifest(build_metadata): |
||||
return { |
||||
"type": "path_tarred", |
||||
"path": "/data/openpilot", |
||||
"casync": { |
||||
"caibx": f"{OPENPILOT_RELEASES}/{build_metadata.canonical}.caibx" |
||||
} |
||||
} |
||||
|
||||
|
||||
def create_manifest(target_dir): |
||||
with open(pathlib.Path(target_dir) / AGNOS_MANIFEST_FILE) as f: |
||||
agnos_manifest = json.load(f) |
||||
|
||||
build_metadata = get_build_metadata(args.target_dir) |
||||
|
||||
return { |
||||
"build_metadata": dataclasses.asdict(build_metadata), |
||||
"manifest": [ |
||||
*[create_partition_manifest(entry) for entry in agnos_manifest], |
||||
create_openpilot_manifest(build_metadata) |
||||
] |
||||
} |
||||
|
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
parser = argparse.ArgumentParser(description="creates a casync release") |
||||
parser.add_argument("target_dir", type=str, help="path to a release build of openpilot to create release from") |
||||
args = parser.parse_args() |
||||
|
||||
target_dir = pathlib.Path(args.target_dir) |
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir: |
||||
casync_dir = pathlib.Path(temp_dir) / "casync" |
||||
casync_dir.mkdir(parents=True) |
||||
|
||||
manifest_file = pathlib.Path(temp_dir) / "manifest.json" |
||||
|
||||
create_casync_caibx(target_dir, casync_dir / "openpilot") |
||||
upload_casync_release(casync_dir) |
||||
manifest = create_manifest(target_dir) |
||||
|
||||
print(json.dumps(manifest, indent=2)) |
||||
@ -1,25 +0,0 @@ |
||||
#!/usr/bin/env python3 |
||||
|
||||
import argparse |
||||
import os |
||||
import pathlib |
||||
from openpilot.tools.lib.azure_container import AzureContainer |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
if "AZURE_TOKEN_OPENPILOT_RELEASES" in os.environ: |
||||
os.environ["AZURE_TOKEN"] = os.environ["AZURE_TOKEN_OPENPILOT_RELEASES"] |
||||
|
||||
OPENPILOT_RELEASES_CONTAINER = AzureContainer("commadist", "openpilot-releases") |
||||
|
||||
parser = argparse.ArgumentParser(description='upload casync folder to azure') |
||||
parser.add_argument("casync_dir", type=str, help="casync directory") |
||||
args = parser.parse_args() |
||||
|
||||
casync_dir = pathlib.Path(args.casync_dir) |
||||
|
||||
for f in casync_dir.rglob("*"): |
||||
if f.is_file(): |
||||
blob_name = f.relative_to(casync_dir) |
||||
print(f"uploading {f} to {blob_name}") |
||||
OPENPILOT_RELEASES_CONTAINER.upload_file(str(f), str(blob_name), overwrite=True) |
||||
Loading…
Reference in new issue