docker: multiarch images (amd64/aarch64) (#29691)

* arch specific tags

* merge multiarch script

* arm64 build and merge jobs

* update tag script

* casual ci test

* change image to 2204

* docker -v test

* arch specific naming

* Remove test_buildjet

* build -arch images only when CURRENT_ARCH_BUILD is set

* support new tags in tag_multiarch

* Toggle CURRENT_ARCH_BUILD on in ci

* Docker common

* Fix condition

* Fix path to docker_common

* Add more stuff to common

* Add build_arm job

* composite job for building

* Run checkout before build composite

* add shell arg

* move timeout to selfdrive_tests

* TARGET ARCHITECTURE var

* Support TARGET_ARCHITECTURE in workflow

* Rewrite to Single build job with matrix

* Remove shebang from docker_common

* Attempt for fix build matrix

* Remove setup arch

* build matrix for docker push

* Use 2vcpu for arm build

* 2vcpu for docker_push too

* temporarly unlock docker_push

* Remove requirement for target arch when pushing

* Unset target architecture in docker_push cl

* fix sha tags

* Rename action to compile-openpilot

* move push_image line to tag_multiarch step

* arch suffix for scons cache

* cache_key_prefix for setup-with-retry

* Re-disable docker_push on non-master

* Add newlines
old-commit-hash: 1344a93a35

.github/workflows/compile-openpilot/action.yaml

@@ -0,0 +1,27 @@
+name: 'compile openpilot'
+
+inputs:
+  cache_key_prefix:
+    description: 'Prefix for caching key'
+    required: false
+    default: 'scons'
+
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      name: Build openpilot with all flags
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+        ${{ env.RUN }} "release/check-dirty.sh"
+    - shell: bash
+      name: Cleanup scons cache and rebuild
+      run: |
+          ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
+                          scons -j$(nproc) --cache-populate"
+    - name: Save scons cache
+      uses: actions/cache/save@v3
+      if: github.ref == 'refs/heads/master'
+      with:
+        path: .ci_cache/scons_cache
+        key: ${{ inputs.cache_key_prefix }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}

.github/workflows/selfdrive_tests.yaml

@@ -67,30 +67,20 @@ jobs:
         cd $STRIPPED_DIR
         ${{ env.RUN }} "unset PYTHONWARNINGS && pre-commit run --all"
 
-  build_all:
+  build:
-    name: build all
+    strategy:
-    runs-on: ubuntu-20.04
+      matrix:
+        arch: ${{ fromJson( (github.repository == 'commaai/openpilot') && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
+    runs-on: ${{ (matrix.arch == 'aarch64') && 'buildjet-2vcpu-ubuntu-2204-arm' || 'ubuntu-20.04' }}
     steps:
     - uses: actions/checkout@v3
       with:
         submodules: true
     - uses: ./.github/workflows/setup-with-retry
-    - name: Build openpilot with all flags
+    - uses: ./.github/workflows/compile-openpilot
-      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 12 || 30) }} # allow more time when we missed the scons cache
+      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
-        ${{ env.RUN }} "release/check-dirty.sh"
-    - name: Cleanup scons cache and rebuild
-      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 2 || 30) }} # allow more time when we missed the scons cache
-      run: |
-          ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
-                          scons -j$(nproc) --cache-populate"
-    - name: Save scons cache
-      uses: actions/cache/save@v3
-      if: github.ref == 'refs/heads/master'
       with:
-        path: .ci_cache/scons_cache
+        cache_key_prefix: scons_${{ matrix.arch }}
-        key: scons-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
 
   build_mac:
     name: build macos
@@ -185,8 +175,10 @@ jobs:
         sudo rm -rf /Applications/ArmGNUToolchain/*/*/.fseventsd
 
   docker_push:
-    name: docker push
+    strategy:
-    runs-on: ubuntu-20.04
+      matrix:
+        arch: ${{ fromJson( (github.repository == 'commaai/openpilot') && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
+    runs-on: ${{ (matrix.arch == 'aarch64') && 'buildjet-2vcpu-ubuntu-2204-arm' || 'ubuntu-20.04' }}
     if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
     steps:
     - uses: actions/checkout@v3
@@ -195,14 +187,34 @@ jobs:
     - name: Setup to push to repo
       run: |
         echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
+        echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV"
         $DOCKER_LOGIN
     - uses: ./.github/workflows/setup-with-retry
       with:
         git-lfs: false
     - name: Build and push CL Docker image
+      if: matrix.arch == 'x86_64'
       run: |
+        echo "TARGET_ARCHITECTURE=''" >> "$GITHUB_ENV"
         eval "$BUILD_CL"
 
+  docker_push_multiarch:
+    name: docker push multiarch tag
+    runs-on: ubuntu-20.04
+    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
+    needs: [docker_push]
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: false
+    - name: Setup docker
+      run: |
+        $DOCKER_LOGIN
+    - name: Merge x64 and arm64 tags
+      run: |
+        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
+        selfdrive/test/docker_tag_multiarch.sh base x86_64 aarch64
+
   static_analysis:
     name: static analysis
     runs-on: ubuntu-20.04

.github/workflows/setup-with-retry/action.yaml

@@ -5,6 +5,10 @@ inputs:
     description: 'Whether or not to pull the git lfs'
     required: false
     default: 'true'
+  cache_key_prefix:
+    description: 'Prefix for caching key'
+    required: false
+    default: 'scons_x86_64'
 
 env:
   SLEEP_TIME: 30 # Time to sleep between retries
@@ -17,6 +21,7 @@ runs:
       continue-on-error: true
       with:
         git_lfs: ${{ inputs.git_lfs }}
+        cache_key_prefix: ${{ inputs.cache_key_prefix }}
         is_retried: true
     - if: steps.setup1.outcome == 'failure'
       shell: bash
@@ -27,6 +32,7 @@ runs:
       continue-on-error: true
       with:
         git_lfs: ${{ inputs.git_lfs }}
+        cache_key_prefix: ${{ inputs.cache_key_prefix }}
         is_retried: true
     - if: steps.setup2.outcome == 'failure'
       shell: bash
@@ -36,4 +42,5 @@ runs:
       uses: ./.github/workflows/setup
       with:
         git_lfs: ${{ inputs.git_lfs }}
+        cache_key_prefix: ${{ inputs.cache_key_prefix }}
         is_retried: true

.github/workflows/setup/action.yaml

@@ -5,6 +5,10 @@ inputs:
     description: 'Whether or not to pull the git lfs'
     required: false
     default: 'true'
+  cache_key_prefix:
+    description: 'Prefix for caching key'
+    required: false
+    default: 'scons_x86_64'
   is_retried:
     description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly'
     required: false
@@ -35,10 +39,10 @@ runs:
       uses: actions/cache/restore@v3
       with:
         path: .ci_cache/scons_cache
-        key: scons-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+        key: ${{ inputs.cache_key_prefix }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
         restore-keys: |
-          scons-${{ env.CACHE_COMMIT_DATE }}-
+          ${{ inputs.cache_key_prefix }}-${{ env.CACHE_COMMIT_DATE }}-
-          scons-
+          ${{ inputs.cache_key_prefix }}-
     # if we didn't get a cache hit, make the directory manually so it doesn't fail on future steps
     - id: scons-cache-setup
       shell: bash

selfdrive/test/docker_build.sh

@@ -1,44 +1,25 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e
 
 # To build sim and docs, you can run the following to mount the scons cache to the same place as in CI:
 # mkdir -p .ci_cache/scons_cache
 # sudo mount --bind /tmp/scons_cache/ .ci_cache/scons_cache
 
-if [ $1 = "base" ]; then
+SCRIPT_DIR=$(dirname "$0")
-  export DOCKER_IMAGE=openpilot-base
+OPENPILOT_DIR=$SCRIPT_DIR/../../
-  export DOCKER_FILE=Dockerfile.openpilot_base
+if [ -n "$TARGET_ARCHITECTURE" ]; then
-elif [ $1 = "docs" ]; then
+  PLATFORM="linux/$TARGET_ARCHITECTURE"
-  export DOCKER_IMAGE=openpilot-docs
+  TAG_SUFFIX="-$TARGET_ARCHITECTURE"
-  export DOCKER_FILE=docs/docker/Dockerfile
-elif [ $1 = "sim" ]; then
-  export DOCKER_IMAGE=openpilot-sim
-  export DOCKER_FILE=tools/sim/Dockerfile.sim
-elif [ $1 = "prebuilt" ]; then
-  export DOCKER_IMAGE=openpilot-prebuilt
-  export DOCKER_FILE=Dockerfile.openpilot
-elif [ $1 = "cl" ]; then
-  export DOCKER_IMAGE=openpilot-base-cl
-  export DOCKER_FILE=Dockerfile.openpilot_base_cl
 else
-  echo "Invalid docker build image $1"
+  PLATFORM="linux/$(uname -m)"
-  exit 1
+  TAG_SUFFIX=""
 fi
 
-export DOCKER_REGISTRY=ghcr.io/commaai
+source $SCRIPT_DIR/docker_common.sh $1 "$TAG_SUFFIX"
-export COMMIT_SHA=$(git rev-parse HEAD)
-
-LOCAL_TAG=$DOCKER_IMAGE
-REMOTE_TAG=$DOCKER_REGISTRY/$LOCAL_TAG
-REMOTE_SHA_TAG=$REMOTE_TAG:$COMMIT_SHA
-
-SCRIPT_DIR=$(dirname "$0")
-OPENPILOT_DIR=$SCRIPT_DIR/../../
 
-DOCKER_BUILDKIT=1 docker buildx build --load --cache-to type=inline --cache-from type=registry,ref=$REMOTE_TAG -t $REMOTE_TAG -t $LOCAL_TAG -f $OPENPILOT_DIR/$DOCKER_FILE $OPENPILOT_DIR
+DOCKER_BUILDKIT=1 docker buildx build --platform $PLATFORM --load --cache-to type=inline --cache-from type=registry,ref=$REMOTE_TAG -t $REMOTE_TAG -t $LOCAL_TAG -f $OPENPILOT_DIR/$DOCKER_FILE $OPENPILOT_DIR
 
-if [[ ! -z "$PUSH_IMAGE" ]];
+if [ -n "$PUSH_IMAGE" ]; then
-then
   docker push $REMOTE_TAG
   docker tag $REMOTE_TAG $REMOTE_SHA_TAG
   docker push $REMOTE_SHA_TAG

selfdrive/test/docker_common.sh

@@ -0,0 +1,27 @@
+if [ $1 = "base" ]; then
+  export DOCKER_IMAGE=openpilot-base
+  export DOCKER_FILE=Dockerfile.openpilot_base
+elif [ $1 = "docs" ]; then
+  export DOCKER_IMAGE=openpilot-docs
+  export DOCKER_FILE=docs/docker/Dockerfile
+elif [ $1 = "sim" ]; then
+  export DOCKER_IMAGE=openpilot-sim
+  export DOCKER_FILE=tools/sim/Dockerfile.sim
+elif [ $1 = "prebuilt" ]; then
+  export DOCKER_IMAGE=openpilot-prebuilt
+  export DOCKER_FILE=Dockerfile.openpilot
+elif [ $1 = "cl" ]; then
+  export DOCKER_IMAGE=openpilot-base-cl
+  export DOCKER_FILE=Dockerfile.openpilot_base_cl
+else
+  echo "Invalid docker build image $1"
+  exit 1
+fi
+
+export DOCKER_REGISTRY=ghcr.io/commaai
+export COMMIT_SHA=$(git rev-parse HEAD)
+
+TAG_SUFFIX=$2
+LOCAL_TAG=$DOCKER_IMAGE$TAG_SUFFIX
+REMOTE_TAG=$DOCKER_REGISTRY/$LOCAL_TAG
+REMOTE_SHA_TAG=$DOCKER_REGISTRY/$LOCAL_TAG:$COMMIT_SHA

selfdrive/test/docker_tag_multiarch.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -e
+
+if [ $# -lt 2 ]; then
+  echo "Usage: $0 <base|docs|sim|prebuilt|cl> <arch1> <arch2> ..."
+  exit 1
+fi
+
+SCRIPT_DIR=$(dirname "$0")
+ARCHS=("${@:2}")
+
+source $SCRIPT_DIR/docker_common.sh $1
+
+MANIFEST_AMENDS=""
+for ARCH in ${ARCHS[@]}; do
+  MANIFEST_AMENDS="$MANIFEST_AMENDS --amend $REMOTE_TAG-$ARCH:$COMMIT_SHA"
+done
+
+docker manifest create $REMOTE_TAG $MANIFEST_AMENDS
+docker manifest create $REMOTE_SHA_TAG $MANIFEST_AMENDS
+
+if [[ -n "$PUSH_IMAGE" ]]; then
+  docker manifest push $REMOTE_TAG
+  docker manifest push $REMOTE_SHA_TAG
+fi