consolidate casync build into fewer scripts and fix pc release build (#32225)

* less scripts

* better

* fixes

* naming

* revert

* cleanup

* lets test it

* fix that one

* and rm

* don't run this

* fix

* not here

* revert testing

* fix docs

* default here too

* t

---------

Co-authored-by: Comma Device <device@comma.ai>
old-commit-hash: 62f053bc4d

Jenkinsfile

@@ -63,6 +63,7 @@ if [ -f /TICI ]; then
   rm -rf /tmp/tmp*
   rm -rf ~/.commacache
   rm -rf /dev/shm/*
+  rm -rf /dev/tmp/tmp*
 
   if ! systemctl is-active --quiet systemd-resolved; then
     echo "restarting resolved"
@@ -179,12 +180,14 @@ def build_git_release(String channel_name) {
 
 
 def build_casync_release(String channel_name, def is_release) {
-  def extra_env = is_release ? "RELEASE=1" : ""
+  def extra_env = is_release ? "RELEASE=1 " : ""
+  def build_dir = "/data/openpilot"
+
+  extra_env += "TMPDIR=/data/tmp PYTHONPATH=$SOURCE_DIR"
 
   return deviceStage("build casync", "tici-needs-can", [], [
-    ["build", "${extra_env} BUILD_DIR=/data/openpilot CASYNC_DIR=/data/casync/openpilot $SOURCE_DIR/release/create_casync_build.sh"],
+    ["build", "${extra_env} $SOURCE_DIR/release/build_release.sh ${build_dir}"],
-    ["create manifest", "$SOURCE_DIR/release/create_release_manifest.py /data/openpilot /data/manifest.json && cat /data/manifest.json"],
+    ["package + upload", "${extra_env} $SOURCE_DIR/release/package_casync_build.py ${build_dir}"],
-    ["upload and cleanup", "PYTHONWARNINGS=ignore $SOURCE_DIR/release/upload_casync_release.py /data/casync && rm -rf /data/casync"],
   ])
 }
 
@@ -199,8 +202,7 @@ def build_stage() {
     },
     'publish agnos': {
       pcStage("publish agnos") {
-        sh "release/create_casync_agnos_release.py /tmp/casync/agnos /tmp/casync_tmp"
+        sh "PYTHONWARNINGS=ignore release/package_casync_agnos.py"
-        sh "PYTHONWARNINGS=ignore ${env.WORKSPACE}/release/upload_casync_release.py /tmp/casync"
       }
     }
   )

SConstruct

@@ -376,11 +376,13 @@ SConscript([
 ])
 if arch != "Darwin":
   SConscript([
-    'system/camerad/SConscript',
     'system/sensord/SConscript',
     'system/logcatd/SConscript',
   ])
 
+if arch == "larch64":
+  SConscript(['system/camerad/SConscript'])
+
 # Build openpilot
 SConscript(['third_party/SConscript'])
 

release/README.md

@@ -4,8 +4,7 @@
 ## terms
 
 - `channel` - a named version of openpilot (git branch, casync caibx) which receives updates
-- `build` - a release which is already built for the comma 3/3x and contains only required files for running openpilot and identifying the release
+- `build` - a copy of openpilot ready for distribution, already built for a specific device
-
 - `build_style` - type of build, either `debug` or `release`
   - `debug` - build with `ALLOW_DEBUG=true`, can test experimental features like longitudinal on alpha cars
   - `release` - build with `ALLOW_DEBUG=false`, experimental features disabled
@@ -22,21 +21,13 @@
 | git branches | `debug`     | installed manually, experimental features enabled, build required |
 
 
-## creating casync build
+## build
-
-`create_casync_build.sh` - creates a casync openpilot build, ready to upload to `openpilot-releases`
 
-```bash
+`release/build_release.sh <build_dir>` - creates an openpilot build into `build_dir`, ready for distribution
-# run on a tici, within the directory you want to create the build from.
-# creates a prebuilt version of openpilot into BUILD_DIR and outputs the caibx
-# of a tarball containing the full prebuilt openpilot release
-BUILD_DIR=/data/openpilot_build    \
-CASYNC_DIR=/data/casync            \
-release/create_casync_build.sh
-```
 
-`upload_casync_release.sh` - helper for uploading a casync build to `openpilot-releases`
+## packaging a casync release
 
+`release/package_casync_build.py <build_dir>` - packages an openpilot build into a casync tar and uploads to `openpilot-releases`
 
 ## release builds
 

release/build_release.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/bash
+
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+SOURCE_DIR="$(git -C $DIR rev-parse --show-toplevel)"
+BUILD_DIR=${1:-$(mktemp -d)}
+
+if [ -f /TICI ]; then
+  FILES_SRC="release/files_tici"
+else
+  FILES_SRC="release/files_pc"
+fi
+
+echo "Building openpilot into $BUILD_DIR"
+
+rm -rf $BUILD_DIR
+mkdir -p $BUILD_DIR
+
+# Copy required files to BUILD_DIR
+cd $SOURCE_DIR
+cp -pR --parents $(cat release/files_common) $BUILD_DIR/
+cp -pR --parents $(cat $FILES_SRC) $BUILD_DIR/
+
+# Build + cleanup
+cd $BUILD_DIR
+export PYTHONPATH="$BUILD_DIR"
+
+rm -f panda/board/obj/panda.bin.signed
+rm -f panda/board/obj/panda_h7.bin.signed
+
+if [ -n "$RELEASE" ]; then
+  export CERT=/data/pandaextra/certs/release
+fi
+
+scons -j$(nproc)
+
+# Cleanup
+find . -name '*.a' -delete
+find . -name '*.o' -delete
+find . -name '*.os' -delete
+find . -name '*.pyc' -delete
+find . -name 'moc_*' -delete
+find . -name '__pycache__' -delete
+rm -rf .sconsign.dblite Jenkinsfile release/
+rm selfdrive/modeld/models/supercombo.onnx
+
+# Mark as prebuilt release
+touch prebuilt
+
+echo "----- openpilot has been built to $BUILD_DIR -----"

release/copy_build_files.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-SOURCE_DIR=$1
-TARGET_DIR=$2
-
-if [ -f /TICI ]; then
-  FILES_SRC="release/files_tici"
-else
-  echo "no release files set"
-  exit 1
-fi
-
-cd $SOURCE_DIR
-cp -pR --parents $(cat release/files_common) $TARGET_DIR/
-cp -pR --parents $(cat $FILES_SRC) $TARGET_DIR/

release/create_casync_build.sh

@@ -1,23 +0,0 @@
-#!/usr/bin/bash
-
-set -ex
-
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
-
-CASYNC_DIR="${CASYNC_DIR:=/tmp/casync}"
-SOURCE_DIR="$(git -C $DIR rev-parse --show-toplevel)"
-BUILD_DIR="${BUILD_DIR:=$(mktemp -d)}"
-PYTHONPATH="$SOURCE_DIR"
-
-echo "Creating casync release from $SOURCE_DIR to $CASYNC_DIR"
-
-cd $SOURCE_DIR
-mkdir -p $CASYNC_DIR
-rm -rf $BUILD_DIR
-mkdir -p $BUILD_DIR
-
-release/copy_build_files.sh $SOURCE_DIR $BUILD_DIR
-release/create_prebuilt.sh $BUILD_DIR
-
-cd $SOURCE_DIR
-release/create_casync_release.py $BUILD_DIR $CASYNC_DIR

release/create_casync_release.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python
-
-import argparse
-import os
-import pathlib
-
-from openpilot.system.updated.casync.common import create_casync_release, create_build_metadata_file
-from openpilot.system.version import get_build_metadata
-
-
-if __name__ == "__main__":
-  parser = argparse.ArgumentParser(description="creates a casync release")
-  parser.add_argument("target_dir", type=str, help="target directory to build channel from")
-  parser.add_argument("output_dir", type=str, help="output directory for the channel")
-  args = parser.parse_args()
-
-  target_dir = pathlib.Path(args.target_dir)
-  output_dir = pathlib.Path(args.output_dir)
-
-  build_metadata = get_build_metadata()
-  build_metadata.openpilot.build_style = "release" if os.environ.get("RELEASE", None) is not None else "debug"
-
-  create_build_metadata_file(target_dir, build_metadata)
-
-  digest, caibx = create_casync_release(target_dir, output_dir, build_metadata.canonical)
-
-  print(f"Created casync release from {target_dir} to {caibx} with digest {digest}")

release/create_prebuilt.sh

@@ -1,34 +0,0 @@
-#!/usr/bin/bash -e
-
-# runs on tici to create a prebuilt version of a release
-
-set -ex
-
-BUILD_DIR=$1
-
-cd $BUILD_DIR
-
-# Build
-export PYTHONPATH="$BUILD_DIR"
-
-rm -f panda/board/obj/panda.bin.signed
-rm -f panda/board/obj/panda_h7.bin.signed
-
-if [ -n "$RELEASE" ]; then
-  export CERT=/data/pandaextra/certs/release
-fi
-
-scons -j$(nproc)
-
-# Cleanup
-find . -name '*.a' -delete
-find . -name '*.o' -delete
-find . -name '*.os' -delete
-find . -name '*.pyc' -delete
-find . -name 'moc_*' -delete
-find . -name '__pycache__' -delete
-rm -rf .sconsign.dblite Jenkinsfile release/
-rm selfdrive/modeld/models/supercombo.onnx
-
-# Mark as prebuilt release
-touch prebuilt

release/create_release_manifest.py

@@ -1,64 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-import dataclasses
-import json
-import os
-import pathlib
-
-from openpilot.system.hardware.tici.agnos import AGNOS_MANIFEST_FILE, get_partition_path
-from openpilot.system.version import get_build_metadata
-
-
-BASE_URL = "https://commadist.blob.core.windows.net"
-
-OPENPILOT_RELEASES = f"{BASE_URL}/openpilot-releases/openpilot"
-AGNOS_RELEASES = f"{BASE_URL}/openpilot-releases/agnos"
-
-
-def create_partition_manifest(partition):
-  agnos_filename = os.path.basename(partition["url"]).split(".")[0]
-
-  return {
-    "type": "partition",
-    "casync": {
-      "caibx": f"{AGNOS_RELEASES}/{agnos_filename}.caibx"
-    },
-    "path": get_partition_path(0, partition),
-    "ab": True,
-    "size": partition["size"],
-    "full_check": partition["full_check"],
-    "hash_raw": partition["hash_raw"],
-  }
-
-
-def create_openpilot_manifest(build_metadata):
-  return {
-    "type": "path_tarred",
-    "path": "/data/openpilot",
-    "casync": {
-      "caibx": f"{OPENPILOT_RELEASES}/{build_metadata.canonical}.caibx"
-    }
-  }
-
-
-if __name__ == "__main__":
-  parser = argparse.ArgumentParser(description="creates a casync release")
-  parser.add_argument("target_dir", type=str, help="directory of the channel to create manifest from")
-  parser.add_argument("output_file", type=str, help="output file to put the manifest")
-  args = parser.parse_args()
-
-  with open(pathlib.Path(args.target_dir) / AGNOS_MANIFEST_FILE) as f:
-    agnos_manifest = json.load(f)
-
-  build_metadata = get_build_metadata(args.target_dir)
-
-  ret = {
-    "build_metadata": dataclasses.asdict(build_metadata),
-    "manifest": [
-      *[create_partition_manifest(entry) for entry in agnos_manifest],
-      create_openpilot_manifest(build_metadata)
-    ]
-  }
-
-  with open(args.output_file, "w") as f:
-    f.write(json.dumps(ret, indent=2))

release/package_casync_agnos.py

@@ -8,27 +8,27 @@ import time
 from openpilot.common.basedir import BASEDIR
 from openpilot.system.hardware.tici.agnos import StreamingDecompressor, unsparsify, noop, AGNOS_MANIFEST_FILE
 from openpilot.system.updated.casync.common import create_casync_from_file
+from release.package_casync_build import upload_casync_release
 
 
 
 if __name__ == "__main__":
   parser = argparse.ArgumentParser(description="creates a casync release")
-  parser.add_argument("output_dir", type=str, help="output directory for the channel")
-  parser.add_argument("working_dir", type=str, help="working directory")
   parser.add_argument("--manifest", type=str, help="json manifest to create agnos release from", \
                         default=str(pathlib.Path(BASEDIR) / AGNOS_MANIFEST_FILE))
   args = parser.parse_args()
 
-  output_dir = pathlib.Path(args.output_dir)
+  manifest_file = pathlib.Path(args.manifest)
-  output_dir.mkdir(parents=True, exist_ok=True)
 
-  working_dir = pathlib.Path(args.working_dir)
+  with tempfile.TemporaryDirectory() as temp_dir:
-  working_dir.mkdir(parents=True, exist_ok=True)
+    working_dir = pathlib.Path(temp_dir)
+    casync_dir = working_dir / "casync"
+    casync_dir.mkdir()
 
-  manifest_file = pathlib.Path(args.manifest)
+    agnos_casync_dir = casync_dir / "agnos"
+    agnos_casync_dir.mkdir()
 
-  with tempfile.NamedTemporaryFile(dir=str(working_dir)) as entry_file:
+    entry_path = working_dir / "entry"
-    entry_path = pathlib.Path(entry_file.name)
 
     with open(manifest_file) as f:
       manifest = json.load(f)
@@ -53,5 +53,7 @@ if __name__ == "__main__":
 
       start = time.monotonic()
       agnos_filename = os.path.basename(entry["url"]).split(".")[0]
-      create_casync_from_file(entry_path, output_dir, agnos_filename)
+      create_casync_from_file(entry_path, agnos_casync_dir, agnos_filename)
       print(f"created casnc in {time.monotonic() - start}")
+
+    upload_casync_release(casync_dir)

release/package_casync_build.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+
+# packages a casync release, uploads to azure, and creates a manifest
+
+import argparse
+import dataclasses
+import json
+import os
+import pathlib
+import tempfile
+
+from openpilot.system.hardware.tici.agnos import AGNOS_MANIFEST_FILE, get_partition_path
+from openpilot.system.updated.casync.common import create_build_metadata_file, create_casync_release
+from openpilot.system.version import get_build_metadata
+from openpilot.tools.lib.azure_container import AzureContainer
+
+
+BASE_URL = "https://commadist.blob.core.windows.net"
+
+OPENPILOT_RELEASES = f"{BASE_URL}/openpilot-releases/openpilot"
+AGNOS_RELEASES = f"{BASE_URL}/openpilot-releases/agnos"
+
+
+def create_casync_caibx(target_dir: pathlib.Path, output_dir: pathlib.Path):
+  output_dir.mkdir()
+  build_metadata = get_build_metadata()
+  build_metadata.openpilot.build_style = "release" if os.environ.get("RELEASE", None) is not None else "debug"
+
+  create_build_metadata_file(target_dir, build_metadata)
+
+  digest, caibx = create_casync_release(target_dir, output_dir, build_metadata.canonical)
+
+  print(f"Created casync release from {target_dir} to {caibx} with digest {digest}")
+
+
+def upload_casync_release(casync_dir: pathlib.Path):
+  if "AZURE_TOKEN_OPENPILOT_RELEASES" in os.environ:
+    os.environ["AZURE_TOKEN"] = os.environ["AZURE_TOKEN_OPENPILOT_RELEASES"]
+
+  OPENPILOT_RELEASES_CONTAINER = AzureContainer("commadist", "openpilot-releases")
+
+  for f in casync_dir.rglob("*"):
+    if f.is_file():
+      blob_name = f.relative_to(casync_dir)
+      print(f"uploading {f} to {blob_name}")
+      OPENPILOT_RELEASES_CONTAINER.upload_file(str(f), str(blob_name), overwrite=True)
+
+
+def create_partition_manifest(partition):
+  agnos_filename = os.path.basename(partition["url"]).split(".")[0]
+
+  return {
+    "type": "partition",
+    "casync": {
+      "caibx": f"{AGNOS_RELEASES}/{agnos_filename}.caibx"
+    },
+    "path": get_partition_path(0, partition),
+    "ab": True,
+    "size": partition["size"],
+    "full_check": partition["full_check"],
+    "hash_raw": partition["hash_raw"],
+  }
+
+
+def create_openpilot_manifest(build_metadata):
+  return {
+    "type": "path_tarred",
+    "path": "/data/openpilot",
+    "casync": {
+      "caibx": f"{OPENPILOT_RELEASES}/{build_metadata.canonical}.caibx"
+    }
+  }
+
+
+def create_manifest(target_dir):
+  with open(pathlib.Path(target_dir) / AGNOS_MANIFEST_FILE) as f:
+    agnos_manifest = json.load(f)
+
+  build_metadata = get_build_metadata(args.target_dir)
+
+  return {
+    "build_metadata": dataclasses.asdict(build_metadata),
+    "manifest": [
+      *[create_partition_manifest(entry) for entry in agnos_manifest],
+      create_openpilot_manifest(build_metadata)
+    ]
+  }
+
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="creates a casync release")
+  parser.add_argument("target_dir", type=str, help="path to a release build of openpilot to create release from")
+  args = parser.parse_args()
+
+  target_dir = pathlib.Path(args.target_dir)
+
+  with tempfile.TemporaryDirectory() as temp_dir:
+    casync_dir = pathlib.Path(temp_dir) / "casync"
+    casync_dir.mkdir(parents=True)
+
+    manifest_file = pathlib.Path(temp_dir) / "manifest.json"
+
+    create_casync_caibx(target_dir, casync_dir / "openpilot")
+    upload_casync_release(casync_dir)
+    manifest = create_manifest(target_dir)
+
+    print(json.dumps(manifest, indent=2))

release/upload_casync_release.py

@@ -1,25 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import os
-import pathlib
-from openpilot.tools.lib.azure_container import AzureContainer
-
-
-if __name__ == "__main__":
-  if "AZURE_TOKEN_OPENPILOT_RELEASES" in os.environ:
-    os.environ["AZURE_TOKEN"] = os.environ["AZURE_TOKEN_OPENPILOT_RELEASES"]
-
-  OPENPILOT_RELEASES_CONTAINER = AzureContainer("commadist", "openpilot-releases")
-
-  parser = argparse.ArgumentParser(description='upload casync folder to azure')
-  parser.add_argument("casync_dir", type=str, help="casync directory")
-  args = parser.parse_args()
-
-  casync_dir = pathlib.Path(args.casync_dir)
-
-  for f in casync_dir.rglob("*"):
-    if f.is_file():
-      blob_name = f.relative_to(casync_dir)
-      print(f"uploading {f} to {blob_name}")
-      OPENPILOT_RELEASES_CONTAINER.upload_file(str(f), str(blob_name), overwrite=True)