CI: use scons cache mount for building simulator and docs (#29436)

* build docker image with scons cache

* use setup

* scons cache

* added the dockerfile

* add the shell

* docs use cache

* dont pull lfs for docs

* simulator too

* dont pull the cache image

* fix inputs

* inputs

* booleans aren't supported

* bool not supported

* bool isnt supported

* fix

* readonly cache

* fix sim cache too

* respect readonly

* gitignore and move to correct folder

* also copy old

* move to tools/ci

* move to selfdrive/test

* just make the cache writable, it's discarded anyway

* minimize diff in this pr

.github/workflows/setup/action.yaml

@@ -1,6 +1,10 @@
 name: 'openpilot env setup'
 
 inputs:
+  setup_docker_scons_cache:
+    description: 'Whether or not to build the scons-cache docker image'
+    required: false
+    default: 'false'
   git_lfs:
     description: 'Whether or not to pull the git lfs'
     required: false
@@ -28,7 +32,14 @@ runs:
         restore-keys: |
           scons-${{ env.CACHE_COMMIT_DATE }}-
           scons-
-
+    - id: setup-scons-cache-docker
+      name: Sets up a docker image with scons cache that can by mounted as a buildkit cache mount
+      shell: bash
+      if: ${{ inputs.setup_docker_scons_cache == 'true' }}
+      run: |
+        cp selfdrive/test/Dockerfile.scons_cache ~
+        cd ~
+        DOCKER_BUILDKIT=1 docker build -t scons-cache -f Dockerfile.scons_cache .
     # build our docker image
     - shell: bash
       run: eval ${{ env.BUILD }}

.github/workflows/tools_tests.yaml

@@ -58,10 +58,14 @@ jobs:
       with:
         submodules: true
     - uses: ./.github/workflows/setup
+      with:
+        setup_docker_scons_cache: true
     - name: Build base cl image
       run: eval "$BUILD_CL"
     - name: Build simulator image
-      run: DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/sim/Dockerfile.sim .
+      run: |
+        docker pull $DOCKER_REGISTRY/$IMAGE_NAME:latest
+        DOCKER_BUILDKIT=1 docker build --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/sim/Dockerfile.sim .
     - name: Push to container registry
       if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
       run: |
@@ -74,14 +78,15 @@ jobs:
     timeout-minutes: 45
     env:
       BUILD: |
-        DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/openpilot-docs-base:latest -t $DOCKER_REGISTRY/openpilot-docs-base:latest -f docs/docker/Dockerfile --target openpilot-docs-base .
+        docker pull $DOCKER_REGISTRY/openpilot-docs:latest
-        DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/openpilot-docs-base:latest --cache-from $DOCKER_REGISTRY/openpilot-docs:latest -t $DOCKER_REGISTRY/openpilot-docs:latest -f docs/docker/Dockerfile .
+        DOCKER_BUILDKIT=1 docker build --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/openpilot-docs:latest -t $DOCKER_REGISTRY/openpilot-docs:latest -f docs/docker/Dockerfile .
     steps:
     - uses: actions/checkout@v3
       with:
         submodules: true
     - uses: ./.github/workflows/setup
       with:
+        setup_docker_scons_cache: true
         git_lfs: false
     - name: Push docker container
       if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'

docs/docker/Dockerfile

@@ -1,4 +1,6 @@
-FROM ghcr.io/commaai/openpilot-base:latest as openpilot-docs-base
+FROM scons-cache as scons-cache
+
+FROM ghcr.io/commaai/openpilot-base:latest
 
 ENV PYTHONUNBUFFERED 1
 
@@ -29,7 +31,7 @@ COPY ./selfdrive ${OPENPILOT_PATH}/selfdrive
 COPY ./system ${OPENPILOT_PATH}/system
 COPY ./*.md ${OPENPILOT_PATH}/
 
-RUN scons -j$(nproc)
+RUN --mount=type=bind,from=scons-cache,source=/tmp/scons_cache,target=/tmp/scons_cache,rw scons -j$(nproc)
 
 RUN apt update && apt install doxygen -y
 COPY ./docs ${OPENPILOT_PATH}/docs
@@ -38,5 +40,5 @@ WORKDIR ${OPENPILOT_PATH}/docs
 RUN make html
 
 FROM nginx:1.21
-COPY --from=0  /home/batman/openpilot/build/docs/html /usr/share/nginx/html
+COPY --from=1 /home/batman/openpilot/build/docs/html /usr/share/nginx/html
 COPY ./docs/docker/nginx.conf /etc/nginx/conf.d/default.conf

selfdrive/test/Dockerfile.scons_cache

@@ -0,0 +1,3 @@
+FROM alpine:3
+
+COPY ./scons_cache /tmp/scons_cache

tools/sim/Dockerfile.sim

@@ -1,3 +1,5 @@
+FROM scons-cache as scons-cache
+
 FROM ghcr.io/commaai/openpilot-base-cl:latest
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -29,6 +31,6 @@ COPY ./system $HOME/openpilot/system
 COPY ./tools $HOME/openpilot/tools
 
 WORKDIR $HOME/openpilot
-RUN scons --cache-readonly -j12
+RUN --mount=type=bind,from=scons-cache,source=/tmp/scons_cache,target=/tmp/scons_cache,rw scons -j12
 
 RUN python -c "from openpilot.selfdrive.test.helpers import set_params_enabled; set_params_enabled()"