From 81b72ee6bae018cfda48580aa717f6ea3139696c Mon Sep 17 00:00:00 2001
From: workinright <workinright@proton.me>
Date: Wed, 27 Aug 2025 00:56:18 +0200
Subject: [PATCH 1/9] 1/2

---
 .github/workflows/badges.yaml           |   3 +-
 .github/workflows/prebuilt.yaml         |   6 +-
 .github/workflows/repo-maintenance.yaml |   4 +-
 .github/workflows/selfdrive_tests.yaml  |   5 +-
 .github/workflows/setup/action.yaml     |  29 ++-
 Dockerfile.openpilot                    |  12 --
 Dockerfile.openpilot_base               |  81 --------
 selfdrive/test/build.sh                 | 257 ++++++++++++++++++++++++
 selfdrive/test/build_inside.sh          |  35 ++++
 selfdrive/test/docker_build.sh          |  26 ---
 selfdrive/test/docker_common.sh         |  18 --
 tools/install_python_dependencies.sh    |   7 +-
 12 files changed, 321 insertions(+), 162 deletions(-)
 delete mode 100644 Dockerfile.openpilot
 delete mode 100644 Dockerfile.openpilot_base
 create mode 100755 selfdrive/test/build.sh
 create mode 100755 selfdrive/test/build_inside.sh
 delete mode 100755 selfdrive/test/docker_build.sh
 delete mode 100644 selfdrive/test/docker_common.sh

diff --git a/.github/workflows/badges.yaml b/.github/workflows/badges.yaml
index 63ee736dca..7b787f77af 100644
--- a/.github/workflows/badges.yaml
+++ b/.github/workflows/badges.yaml
@@ -6,8 +6,7 @@ on:
 
 env:
   BASE_IMAGE: openpilot-base
-  DOCKER_REGISTRY: ghcr.io/commaai
-  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c
+  RUN: export UV_BIN="/home/runner/.local/bin" ; export PATH="$UV_BIN:$PATH" ; source /home/runner/.venv/bin/activate || true ; export PYTHONPATH=/tmp/openpilot ; export CI=1 ; export PYTHONWARNINGS=error ; export FILEREADER_CACHE=1 ; bash -c
 
 jobs:
   badges:
diff --git a/.github/workflows/prebuilt.yaml b/.github/workflows/prebuilt.yaml
index d8963ec89f..f4a5967253 100644
--- a/.github/workflows/prebuilt.yaml
+++ b/.github/workflows/prebuilt.yaml
@@ -5,8 +5,7 @@ on:
   workflow_dispatch:
 
 env:
-  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: selfdrive/test/docker_build.sh prebuilt
+  BUILD: selfdrive/test/build.sh prebuilt
 
 jobs:
   build_prebuilt:
@@ -33,7 +32,6 @@ jobs:
       with:
         submodules: true
     - run: git lfs pull
-    - name: Build and Push docker image
+    - name: Build and Push rootfs image
       run: |
-        $DOCKER_LOGIN
         eval "$BUILD"
diff --git a/.github/workflows/repo-maintenance.yaml b/.github/workflows/repo-maintenance.yaml
index 7bb91c0ca4..8a9f8215ae 100644
--- a/.github/workflows/repo-maintenance.yaml
+++ b/.github/workflows/repo-maintenance.yaml
@@ -7,8 +7,8 @@ on:
 
 env:
   BASE_IMAGE: openpilot-base
-  BUILD: selfdrive/test/docker_build.sh base
-  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
+  BUILD: selfdrive/test/build.sh base
+  RUN: export UV_BIN="/home/runner/.local/bin" ; export PATH="$UV_BIN:$PATH" ; source /home/runner/.venv/bin/activate || true ; export PYTHONPATH=/tmp/openpilot ; export CI=1 ; export PYTHONWARNINGS=error ; export FILEREADER_CACHE=1 ; bash -c
 
 jobs:
   update_translations:
diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml
index aad772757d..44ff010862 100644
--- a/.github/workflows/selfdrive_tests.yaml
+++ b/.github/workflows/selfdrive_tests.yaml
@@ -22,10 +22,9 @@ env:
   BASE_IMAGE: openpilot-base
   AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}
 
-  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: selfdrive/test/docker_build.sh base
+  BUILD: selfdrive/test/build.sh base
 
-  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
+  RUN: export UV_BIN="/home/runner/.local/bin" ; export PATH="$UV_BIN:$PATH" ; source /home/runner/.venv/bin/activate || true ; export PYTHONPATH=/tmp/openpilot ; export CI=1 ; export PYTHONWARNINGS=error ; export FILEREADER_CACHE=1 ; bash -c
 
   PYTEST: pytest --continue-on-collection-errors --durations=0 -n logical
 
diff --git a/.github/workflows/setup/action.yaml b/.github/workflows/setup/action.yaml
index 818060c3b0..a21616dab1 100644
--- a/.github/workflows/setup/action.yaml
+++ b/.github/workflows/setup/action.yaml
@@ -30,12 +30,12 @@ runs:
     - shell: bash
       run: git lfs pull
 
-    # build cache
+    # build scons cache
     - id: date
       shell: bash
-      run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
+      run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV ; echo "CACHE_FILE_HASHES=$(sha256sum selfdrive/test/build.sh tools/install_ubuntu_dependencies.sh tools/install_python_dependencies.sh | cut -d" " -f1 | tr '\n' '_')" >> $GITHUB_ENV
     - shell: bash
-      run: echo "$CACHE_COMMIT_DATE"
+      run: echo "$CACHE_COMMIT_DATE" ; echo "$CACHE_FILE_HASHES"
     - id: scons-cache
       uses: ./.github/workflows/auto-cache
       with:
@@ -44,13 +44,20 @@ runs:
         restore-keys: |
           scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}
           scons-${{ runner.arch }}
-    # as suggested here: https://github.com/moby/moby/issues/32816#issuecomment-910030001
-    - id: normalize-file-permissions
-      shell: bash
-      name: Normalize file permissions to ensure a consistent docker build cache
-      run: |
-        find . -type f -executable -not -perm 755 -exec chmod 755 {} \;
-        find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \;
-    # build our docker image
+
+    # try to restore rootfs image cache
+    - id: rootfs-cache
+      uses: actions/cache/restore@v4
+      with:
+        path: /tmp/rootfs_cache.tar
+        key: rootfs-${{ runner.arch }}-${{ env.CACHE_FILE_HASHES }}
+    # build our rootfs image
     - shell: bash
       run: eval ${{ env.BUILD }}
+    # try to save rootfs image cache
+    - name: Save rootfs cache
+      uses: actions/cache/save@v4
+      #if: github.ref == 'refs/heads/master'
+      with:
+        path: /tmp/rootfs_cache.tar
+        key: rootfs-${{ runner.arch }}-${{ env.CACHE_FILE_HASHES }}
diff --git a/Dockerfile.openpilot b/Dockerfile.openpilot
deleted file mode 100644
index d85be77121..0000000000
--- a/Dockerfile.openpilot
+++ /dev/null
@@ -1,12 +0,0 @@
-FROM ghcr.io/commaai/openpilot-base:latest
-
-ENV PYTHONUNBUFFERED=1
-
-ENV OPENPILOT_PATH=/home/batman/openpilot
-
-RUN mkdir -p ${OPENPILOT_PATH}
-WORKDIR ${OPENPILOT_PATH}
-
-COPY . ${OPENPILOT_PATH}/
-
-RUN scons --cache-readonly -j$(nproc)
diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base
deleted file mode 100644
index 44d8d95e95..0000000000
--- a/Dockerfile.openpilot_base
+++ /dev/null
@@ -1,81 +0,0 @@
-FROM ubuntu:24.04
-
-ENV PYTHONUNBUFFERED=1
-
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot python3-tk python3-dev && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US:en
-ENV LC_ALL=en_US.UTF-8
-
-COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
-RUN /tmp/tools/install_ubuntu_dependencies.sh && \
-    rm -rf /var/lib/apt/lists/* /tmp/* && \
-    cd /usr/lib/gcc/arm-none-eabi/* && \
-    rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
-
-# Add OpenCL
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    apt-utils \
-    alien \
-    unzip \
-    tar \
-    curl \
-    xz-utils \
-    dbus \
-    gcc-arm-none-eabi \
-    tmux \
-    vim \
-    libx11-6 \
-    wget \
-  && rm -rf /var/lib/apt/lists/*
-
-RUN mkdir -p /tmp/opencl-driver-intel && \
-    cd /tmp/opencl-driver-intel && \
-    wget https://github.com/intel/llvm/releases/download/2024-WW14/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
-    wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.12.0/oneapi-tbb-2021.12.0-lin.tgz && \
-    mkdir -p /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
-    cd /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
-    tar -zxvf /tmp/opencl-driver-intel/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
-    mkdir -p /etc/OpenCL/vendors && \
-    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64/libintelocl.so > /etc/OpenCL/vendors/intel_expcpu.icd && \
-    cd /opt/intel && \
-    tar -zxvf /tmp/opencl-driver-intel/oneapi-tbb-2021.12.0-lin.tgz && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so.12 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so.2 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
-    mkdir -p /etc/ld.so.conf.d && \
-    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf && \
-    ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf && \
-    cd / && \
-    rm -rf /tmp/opencl-driver-intel
-
-ENV NVIDIA_VISIBLE_DEVICES=all
-ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
-ENV QTWEBENGINE_DISABLE_SANDBOX=1
-
-RUN dbus-uuidgen > /etc/machine-id
-
-ARG USER=batman
-ARG USER_UID=1001
-RUN useradd -m -s /bin/bash -u $USER_UID $USER
-RUN usermod -aG sudo $USER
-RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-USER $USER
-
-COPY --chown=$USER pyproject.toml uv.lock /home/$USER
-COPY --chown=$USER tools/install_python_dependencies.sh /home/$USER/tools/
-
-ENV VIRTUAL_ENV=/home/$USER/.venv
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-RUN cd /home/$USER && \
-    tools/install_python_dependencies.sh && \
-    rm -rf tools/ pyproject.toml uv.lock .cache
-
-USER root
-RUN sudo git config --global --add safe.directory /tmp/openpilot
diff --git a/selfdrive/test/build.sh b/selfdrive/test/build.sh
new file mode 100755
index 0000000000..ddcde537c8
--- /dev/null
+++ b/selfdrive/test/build.sh
@@ -0,0 +1,257 @@
+#!/usr/bin/env bash
+#set -e
+trap restore_roota ERR
+ORG_PWD="$PWD"
+
+REPO="$HOME/work/openpilot/openpilot"
+CACHE_ROOTFS_TARBALL_PATH="/tmp/rootfs_cache.tar"
+
+prepare_mounts() {
+    # create and mount the required volumes where they're expected
+    mkdir -p /tmp/openpilot /tmp/scons_cache /tmp/comma_download_cache /tmp/openpilot_cache
+    sudo mount --bind "$REPO" /tmp/openpilot
+
+    sudo mount --bind "$REPO/.ci_cache/scons_cache" /tmp/scons_cache || true
+    sudo mount --bind "$REPO/.ci_cache/comma_download_cache" /tmp/comma_download_cache || true
+    sudo mount --bind "$REPO/.ci_cache/openpilot_cache" /tmp/openpilot_cache || true
+
+    # needed for the unit tests not to fail
+    sudo chmod 755 /sys/fs/pstore
+}
+
+commit_root() {
+
+    sudo mkdir -p /base /newroot /upper /work
+    
+    sudo unshare -f --kill-child -m $ORG_PWD/selfdrive/test/build_inside.sh
+    ec=$?
+    echo "end of ns"
+
+    sudo rm -rf /base /newroot /work
+
+    mkdir -p /tmp/rootfs_cache
+    sudo rm -f "$CACHE_ROOTFS_TARBALL_PATH" # remove the old tarball from previous run, if exists
+    cd /upper
+    sudo tar -cf "$CACHE_ROOTFS_TARBALL_PATH" .
+    cd
+
+    sudo rm -rf /upper
+
+    cd /
+    sudo tar -xf "$CACHE_ROOTFS_TARBALL_PATH" 2>/dev/null || true
+    cd
+
+    prepare_mounts
+
+    exit $ec
+
+
+}
+
+#declare -a mounts
+commit_root_old() {
+
+    sudo mkdir -p /upper /work /overlay # prepare directories for overlayfs
+    org_mounts="$(cat /proc/mounts)" # save the original mounts table
+
+    target="/overlay"
+    sudo mkdir -p /lower
+    sudo mount --bind / /lower
+    mounts+=("/lower")
+    sudo mount -t overlay overlay -o lowerdir=/lower,upperdir=/upper,workdir=/work "$target" # mount the overlayfs
+    mounts+=("/")
+
+    while read line # bind-mount any mounts under the old rootfs into the new one (overlayfs isn't recursive like e.g. `mount --rbind`)
+    do
+        echo DOING $line
+        if [ "$line" != "/" ] # except the rootfs base, to avoid infinite mountpoint loops
+        then
+            target="/overlay$line"
+            sudo mount --bind "$line" "$target" || true
+            mounts+=("$line")
+        fi
+    done < <(echo "$org_mounts" | cut -d" " -f2)
+
+    # remove the MS_SHARED flag from the original rootfs mount, which isn't supported by pivot_root(8) and would cause it to fail (see: https://lxc-users.linuxcontainers.narkive.com/pNQKxcnN/pivot-root-failures-when-is-mounted-as-shared)
+    sudo mount --make-rprivate /
+
+    # prepare for the pivot_root(8) and execute, swapping places of the original rootfs and the new one on overlayfs (with its lowerdir still being the original one)
+    # (what this achieves is committing the state of the original rootfs and making it read-only, while creating a new, virtual read-write rootfs with all changes written into a separate directory, the upperdir)
+    cd /overlay
+    sudo mkdir -p old
+    sudo pivot_root . old # once this finishes, the system is moved to the new rootfs and all newly open file descriptors will point to it
+    #sudo systemctl daemon-reexec
+    cd
+
+    mount
+
+    ls /home/runner
+
+    sudo touch /root_commited
+
+    exec "$ORG_PWD/$0" "$@"
+
+
+}
+
+restore_roota() {
+
+    echo failed at ${BASH_LINENO[0]}
+
+}
+
+restore_root() {
+
+    echo failed at ${BASH_LINENO[0]}
+
+    echo mounts "${mounts[@]}"
+
+    cd /old
+    sudo mkdir -p new
+    sudo pivot_root . new
+    #sudo systemctl daemon-reexec
+    cd
+
+
+    for (( i=${#mounts[@]}-1; i>=0; i-- ))
+    do
+        #sudo lsof "/new/${mounts[i]}" || true
+        sudo umount -l "/lower/overlay/${mounts[i]}" || true
+        sudo umount -l "/new/${mounts[i]}" || true
+    done
+
+    mount
+
+    rm -f /root_commited
+
+}
+
+echo AAA
+mount
+echo BBB
+
+if [ -f "$CACHE_ROOTFS_TARBALL_PATH" ]
+then
+    # if the rootfs diff tarball (also created by this script) got restored from the CI native cache, unpack it, upgrading the rootfs
+    echo "restoring rootfs from the native build cache"
+    cd /
+    sudo tar -xf "$CACHE_ROOTFS_TARBALL_PATH" 2>/dev/null || true
+    cd
+    rm "$CACHE_ROOTFS_TARBALL_PATH"
+
+    # before the next tasks are run, finalize the environment for them
+    prepare_mounts
+
+    # EXITS HERE - if the rootfs could been prepared entirely from the cache, there's no need for any further action like re-building
+    exit 0
+else
+    # otherwise, we'll have to install everything from scratch and build the tarball to be available for the next run
+    echo "no native build cache entry restored, rebuilding"
+fi
+
+# in case this script was run on the same instance before, umount any overlays which were mounted by the previous runs
+#tac /proc/mounts | grep overlay | grep -v "overlay / " | cut -d" " -f2 | while read line; do sudo umount "$line" || true; done
+
+# in order to be able to build a diff rootfs tarball, we need to commit its initial state by moving it on-the-fly to overlayfs;
+# below, we prepare the system and the new rootfs itself
+
+
+if ! [ -e /root_committed ]
+then
+commit_root
+fi
+
+
+
+# -------- at this point, the original rootfs was committed and all the changes to it done below will be saved to the newly created rootfs diff tarball --------
+
+# install and set up the native dependencies needed
+PYTHONUNBUFFERED=1
+DEBIAN_FRONTEND=noninteractive
+
+mkdir -p /tmp/tools
+cp "$REPO/tools/install_ubuntu_dependencies.sh" /tmp/tools/
+sudo /tmp/tools/install_ubuntu_dependencies.sh &>/dev/null
+
+sudo apt-get install -y --no-install-recommends \
+    sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot python3-tk python3-dev \
+    apt-utils alien unzip tar curl xz-utils dbus gcc-arm-none-eabi tmux vim libx11-6 wget &>/dev/null
+
+sudo rm -rf /var/lib/apt/lists/*
+sudo apt-get clean
+
+cd /usr/lib/gcc/arm-none-eabi/*
+sudo rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
+cd
+
+sudo sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen
+sudo locale-gen
+LANG=en_US.UTF-8
+LANGUAGE=en_US:en
+LC_ALL=en_US.UTF-8
+
+mkdir -p /tmp/opencl-driver-intel
+cd /tmp/opencl-driver-intel
+wget https://github.com/intel/llvm/releases/download/2024-WW14/oclcpuexp-2024.17.3.0.09_rel.tar.gz &>/dev/null
+wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.12.0/oneapi-tbb-2021.12.0-lin.tgz &>/dev/null
+sudo mkdir -p /opt/intel/oclcpuexp_2024.17.3.0.09_rel
+cd /opt/intel/oclcpuexp_2024.17.3.0.09_rel
+sudo tar -zxvf /tmp/opencl-driver-intel/oclcpuexp-2024.17.3.0.09_rel.tar.gz
+sudo mkdir -p /etc/OpenCL/vendors
+sudo bash -c "echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64/libintelocl.so > /etc/OpenCL/vendors/intel_expcpu.icd"
+cd /opt/intel
+sudo tar -zxvf /tmp/opencl-driver-intel/oneapi-tbb-2021.12.0-lin.tgz
+sudo ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64
+sudo ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64
+sudo ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so.12 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64
+sudo ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so.2 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64
+sudo mkdir -p /etc/ld.so.conf.d
+sudo bash -c "echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf"
+sudo ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf
+cd /
+rm -rf /tmp/opencl-driver-intel
+cd
+
+sudo bash -c "dbus-uuidgen > /etc/machine-id"
+
+NVIDIA_VISIBLE_DEVICES=all
+NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
+QTWEBENGINE_DISABLE_SANDBOX=1
+
+# install and set up the Python dependencies needed
+
+#sudo useradd -m -s /bin/bash -u 1002 test
+sudo cp "/home/runner/work/openpilot/openpilot/pyproject.toml" "/home/runner/work/openpilot/openpilot/uv.lock" "/home/runner/work/openpilot/openpilot/tools/install_python_dependencies.sh" \
+    /home/runner/
+#sudo chown -R test:test /home/test
+
+cd
+#chown -R runner:run
+
+rm -rf .venv
+
+mkdir aaa
+cd aaa
+../install_python_dependencies.sh
+cd
+
+
+#sudo -u test bash -c " export HOME=/home/test ; export XDG_CONFIG_HOME=/home/test/.config ; cd /home/test ; ./install_python_dependencies.sh"
+#sudo chown -R runner:runner /home/test
+
+#cd /home/test ; rm pyproject.toml uv.lock install_python_dependencies.sh ; ls -la ; cd
+rm pyproject.toml uv.lock install_python_dependencies.sh
+
+#sudo rsync -aL /home/test/.venv/ /home/runner/.venv/
+#sudo rsync -aL /home/test/.local/ /home/runner/.local/
+#sudo rm -rf /home/test
+
+
+# add a git safe directory for compiling openpilot
+sudo git config --global --add safe.directory /tmp/openpilot
+
+# finally, create the rootfs diff tarball (to be pushed into the CI native cache)
+
+
+# before the next tasks are run, finalize the environment for them
+#prepare_mounts
diff --git a/selfdrive/test/build_inside.sh b/selfdrive/test/build_inside.sh
new file mode 100755
index 0000000000..2436e1eee6
--- /dev/null
+++ b/selfdrive/test/build_inside.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+sudo mount --bind / /base
+mount -t overlay overlay -o lowerdir=/base,upperdir=/upper,workdir=/work /newroot
+rm -f /newroot/etc/resolv.conf
+touch /newroot/etc/resolv.conf
+cat /etc/resolv.conf > /newroot/etc/resolv.conf
+
+
+
+mkdir -p /newroot/old
+cd /newroot
+pivot_root . old
+
+mount -t proc proc /proc
+mount -t devtmpfs devtmpfs /dev
+mkdir -p /dev/pts
+mount -t devpts devpts /dev/pts
+mount -t proc proc /proc
+mount -t sysfs sysfs /sys
+
+
+
+mount
+
+touch /root_committed
+sudo -u runner /home/runner/work/openpilot/openpilot/selfdrive/test/build.sh
+ec=$?
+
+echo "hello"
+
+#sudo umount /newroot
+#sudo umount /base
+
+exit $ec
diff --git a/selfdrive/test/docker_build.sh b/selfdrive/test/docker_build.sh
deleted file mode 100755
index 4d58a1507c..0000000000
--- a/selfdrive/test/docker_build.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-# To build sim and docs, you can run the following to mount the scons cache to the same place as in CI:
-# mkdir -p .ci_cache/scons_cache
-# sudo mount --bind /tmp/scons_cache/ .ci_cache/scons_cache
-
-SCRIPT_DIR=$(dirname "$0")
-OPENPILOT_DIR=$SCRIPT_DIR/../../
-if [ -n "$TARGET_ARCHITECTURE" ]; then
-  PLATFORM="linux/$TARGET_ARCHITECTURE"
-  TAG_SUFFIX="-$TARGET_ARCHITECTURE"
-else
-  PLATFORM="linux/$(uname -m)"
-  TAG_SUFFIX=""
-fi
-
-source $SCRIPT_DIR/docker_common.sh $1 "$TAG_SUFFIX"
-
-DOCKER_BUILDKIT=1 docker buildx build --provenance false --pull --platform $PLATFORM --load --cache-to type=inline --cache-from type=registry,ref=$REMOTE_TAG -t $DOCKER_IMAGE:latest -t $REMOTE_TAG -t $LOCAL_TAG -f $OPENPILOT_DIR/$DOCKER_FILE $OPENPILOT_DIR
-
-if [ -n "$PUSH_IMAGE" ]; then
-  docker push $REMOTE_TAG
-  docker tag $REMOTE_TAG $REMOTE_SHA_TAG
-  docker push $REMOTE_SHA_TAG
-fi
diff --git a/selfdrive/test/docker_common.sh b/selfdrive/test/docker_common.sh
deleted file mode 100644
index 2887fff74b..0000000000
--- a/selfdrive/test/docker_common.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-if [ "$1" = "base" ]; then
-  export DOCKER_IMAGE=openpilot-base
-  export DOCKER_FILE=Dockerfile.openpilot_base
-elif [ "$1" = "prebuilt" ]; then
-  export DOCKER_IMAGE=openpilot-prebuilt
-  export DOCKER_FILE=Dockerfile.openpilot
-else
-  echo "Invalid docker build image: '$1'"
-  exit 1
-fi
-
-export DOCKER_REGISTRY=ghcr.io/commaai
-export COMMIT_SHA=$(git rev-parse HEAD)
-
-TAG_SUFFIX=$2
-LOCAL_TAG=$DOCKER_IMAGE$TAG_SUFFIX
-REMOTE_TAG=$DOCKER_REGISTRY/$LOCAL_TAG
-REMOTE_SHA_TAG=$DOCKER_REGISTRY/$LOCAL_TAG:$COMMIT_SHA
diff --git a/tools/install_python_dependencies.sh b/tools/install_python_dependencies.sh
index cdbaca32cf..6fa8138755 100755
--- a/tools/install_python_dependencies.sh
+++ b/tools/install_python_dependencies.sh
@@ -5,8 +5,9 @@ set -e
 export PIP_DEFAULT_TIMEOUT=200
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-ROOT="$DIR"/../
-cd "$ROOT"
+echo DIR "$DIR"
+#ROOT="$DIR"/../
+#cd "$ROOT"
 
 if ! command -v "uv" > /dev/null 2>&1; then
   echo "installing uv..."
@@ -21,7 +22,7 @@ uv self update || true
 
 echo "installing python packages..."
 uv sync --frozen --all-extras
-source .venv/bin/activate
+source $HOME/.venv/bin/activate
 
 if [[ "$(uname)" == 'Darwin' ]]; then
   touch "$ROOT"/.env

From 3fe703e03df51040724860977f0ecd42bfa5171a Mon Sep 17 00:00:00 2001
From: workinright <workinright@proton.me>
Date: Mon, 1 Sep 2025 07:21:45 +0200
Subject: [PATCH 2/9] 1/2

---
 selfdrive/test/build.sh        | 160 ++++++++++-----------------------
 selfdrive/test/build_inside.sh |  35 --------
 2 files changed, 45 insertions(+), 150 deletions(-)
 delete mode 100755 selfdrive/test/build_inside.sh

diff --git a/selfdrive/test/build.sh b/selfdrive/test/build.sh
index ddcde537c8..5ebea40ebe 100755
--- a/selfdrive/test/build.sh
+++ b/selfdrive/test/build.sh
@@ -1,29 +1,21 @@
 #!/usr/bin/env bash
 #set -e
-trap restore_roota ERR
+trap restore_root ERR
 ORG_PWD="$PWD"
 
 REPO="$HOME/work/openpilot/openpilot"
 CACHE_ROOTFS_TARBALL_PATH="/tmp/rootfs_cache.tar"
 
-prepare_mounts() {
-    # create and mount the required volumes where they're expected
-    mkdir -p /tmp/openpilot /tmp/scons_cache /tmp/comma_download_cache /tmp/openpilot_cache
-    sudo mount --bind "$REPO" /tmp/openpilot
-
-    sudo mount --bind "$REPO/.ci_cache/scons_cache" /tmp/scons_cache || true
-    sudo mount --bind "$REPO/.ci_cache/comma_download_cache" /tmp/comma_download_cache || true
-    sudo mount --bind "$REPO/.ci_cache/openpilot_cache" /tmp/openpilot_cache || true
-
-    # needed for the unit tests not to fail
-    sudo chmod 755 /sys/fs/pstore
+unpack_rootfs_tarball() {
+    cd /
+    sudo tar -xf "$CACHE_ROOTFS_TARBALL_PATH" 2>/dev/null || true
+    cd
 }
 
 commit_root() {
-
     sudo mkdir -p /base /newroot /upper /work
     
-    sudo unshare -f --kill-child -m $ORG_PWD/selfdrive/test/build_inside.sh
+    sudo unshare -f --kill-child -m $ORG_PWD/selfdrive/test/build.sh build_inside_namespace
     ec=$?
     echo "end of ns"
 
@@ -37,106 +29,65 @@ commit_root() {
 
     sudo rm -rf /upper
 
-    cd /
-    sudo tar -xf "$CACHE_ROOTFS_TARBALL_PATH" 2>/dev/null || true
-    cd
+    unpack_rootfs_tarball
 
     prepare_mounts
 
     exit $ec
-
-
-}
-
-#declare -a mounts
-commit_root_old() {
-
-    sudo mkdir -p /upper /work /overlay # prepare directories for overlayfs
-    org_mounts="$(cat /proc/mounts)" # save the original mounts table
-
-    target="/overlay"
-    sudo mkdir -p /lower
-    sudo mount --bind / /lower
-    mounts+=("/lower")
-    sudo mount -t overlay overlay -o lowerdir=/lower,upperdir=/upper,workdir=/work "$target" # mount the overlayfs
-    mounts+=("/")
-
-    while read line # bind-mount any mounts under the old rootfs into the new one (overlayfs isn't recursive like e.g. `mount --rbind`)
-    do
-        echo DOING $line
-        if [ "$line" != "/" ] # except the rootfs base, to avoid infinite mountpoint loops
-        then
-            target="/overlay$line"
-            sudo mount --bind "$line" "$target" || true
-            mounts+=("$line")
-        fi
-    done < <(echo "$org_mounts" | cut -d" " -f2)
-
-    # remove the MS_SHARED flag from the original rootfs mount, which isn't supported by pivot_root(8) and would cause it to fail (see: https://lxc-users.linuxcontainers.narkive.com/pNQKxcnN/pivot-root-failures-when-is-mounted-as-shared)
-    sudo mount --make-rprivate /
-
-    # prepare for the pivot_root(8) and execute, swapping places of the original rootfs and the new one on overlayfs (with its lowerdir still being the original one)
-    # (what this achieves is committing the state of the original rootfs and making it read-only, while creating a new, virtual read-write rootfs with all changes written into a separate directory, the upperdir)
-    cd /overlay
-    sudo mkdir -p old
-    sudo pivot_root . old # once this finishes, the system is moved to the new rootfs and all newly open file descriptors will point to it
-    #sudo systemctl daemon-reexec
-    cd
-
-    mount
-
-    ls /home/runner
-
-    sudo touch /root_commited
-
-    exec "$ORG_PWD/$0" "$@"
-
-
 }
 
-restore_roota() {
+prepare_mounts() {
+    # create and mount the required volumes where they're expected
+    mkdir -p /tmp/openpilot /tmp/scons_cache /tmp/comma_download_cache /tmp/openpilot_cache
+    sudo mount --bind "$REPO" /tmp/openpilot
 
-    echo failed at ${BASH_LINENO[0]}
+    sudo mount --bind "$REPO/.ci_cache/scons_cache" /tmp/scons_cache || true
+    sudo mount --bind "$REPO/.ci_cache/comma_download_cache" /tmp/comma_download_cache || true
+    sudo mount --bind "$REPO/.ci_cache/openpilot_cache" /tmp/openpilot_cache || true
 
+    # needed for the unit tests not to fail
+    sudo chmod 755 /sys/fs/pstore
 }
 
 restore_root() {
-
     echo failed at ${BASH_LINENO[0]}
+}
 
-    echo mounts "${mounts[@]}"
-
-    cd /old
-    sudo mkdir -p new
-    sudo pivot_root . new
-    #sudo systemctl daemon-reexec
-    cd
-
-
-    for (( i=${#mounts[@]}-1; i>=0; i-- ))
-    do
-        #sudo lsof "/new/${mounts[i]}" || true
-        sudo umount -l "/lower/overlay/${mounts[i]}" || true
-        sudo umount -l "/new/${mounts[i]}" || true
-    done
-
-    mount
-
-    rm -f /root_commited
-
+build_inside_namespace() {
+    mount --bind / /base
+    mount -t overlay overlay -o lowerdir=/base,upperdir=/upper,workdir=/work /newroot
+    rm -f /newroot/etc/resolv.conf
+    touch /newroot/etc/resolv.conf
+    cat /etc/resolv.conf > /newroot/etc/resolv.conf
+
+    mkdir -p /newroot/old
+    cd /newroot
+    pivot_root . old
+
+    mount -t proc proc /proc
+    mount -t devtmpfs devtmpfs /dev
+    mkdir -p /dev/pts
+    mount -t devpts devpts /dev/pts
+    mount -t proc proc /proc
+    mount -t sysfs sysfs /sys
+
+    touch /root_committed
+    sudo -u runner /home/runner/work/openpilot/openpilot/selfdrive/test/build.sh
+    ec=$?
+    exit $ec
 }
 
-echo AAA
-mount
-echo BBB
+if [ "$1" = "build_inside_namespace" ]
+then
+    build_inside_namespace
+    exit
+fi
 
 if [ -f "$CACHE_ROOTFS_TARBALL_PATH" ]
 then
     # if the rootfs diff tarball (also created by this script) got restored from the CI native cache, unpack it, upgrading the rootfs
     echo "restoring rootfs from the native build cache"
-    cd /
-    sudo tar -xf "$CACHE_ROOTFS_TARBALL_PATH" 2>/dev/null || true
-    cd
+    unpack_rootfs_tarball
     rm "$CACHE_ROOTFS_TARBALL_PATH"
 
     # before the next tasks are run, finalize the environment for them
@@ -149,20 +100,14 @@ else
     echo "no native build cache entry restored, rebuilding"
 fi
 
-# in case this script was run on the same instance before, umount any overlays which were mounted by the previous runs
-#tac /proc/mounts | grep overlay | grep -v "overlay / " | cut -d" " -f2 | while read line; do sudo umount "$line" || true; done
-
 # in order to be able to build a diff rootfs tarball, we need to commit its initial state by moving it on-the-fly to overlayfs;
 # below, we prepare the system and the new rootfs itself
 
-
 if ! [ -e /root_committed ]
 then
 commit_root
 fi
 
-
-
 # -------- at this point, the original rootfs was committed and all the changes to it done below will be saved to the newly created rootfs diff tarball --------
 
 # install and set up the native dependencies needed
@@ -219,33 +164,18 @@ NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
 QTWEBENGINE_DISABLE_SANDBOX=1
 
 # install and set up the Python dependencies needed
-
-#sudo useradd -m -s /bin/bash -u 1002 test
 sudo cp "/home/runner/work/openpilot/openpilot/pyproject.toml" "/home/runner/work/openpilot/openpilot/uv.lock" "/home/runner/work/openpilot/openpilot/tools/install_python_dependencies.sh" \
     /home/runner/
-#sudo chown -R test:test /home/test
 
 cd
-#chown -R runner:run
-
 rm -rf .venv
 
 mkdir aaa
 cd aaa
 ../install_python_dependencies.sh
 cd
-
-
-#sudo -u test bash -c " export HOME=/home/test ; export XDG_CONFIG_HOME=/home/test/.config ; cd /home/test ; ./install_python_dependencies.sh"
-#sudo chown -R runner:runner /home/test
-
-#cd /home/test ; rm pyproject.toml uv.lock install_python_dependencies.sh ; ls -la ; cd
 rm pyproject.toml uv.lock install_python_dependencies.sh
 
-#sudo rsync -aL /home/test/.venv/ /home/runner/.venv/
-#sudo rsync -aL /home/test/.local/ /home/runner/.local/
-#sudo rm -rf /home/test
-
 
 # add a git safe directory for compiling openpilot
 sudo git config --global --add safe.directory /tmp/openpilot
diff --git a/selfdrive/test/build_inside.sh b/selfdrive/test/build_inside.sh
deleted file mode 100755
index 2436e1eee6..0000000000
--- a/selfdrive/test/build_inside.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-
-sudo mount --bind / /base
-mount -t overlay overlay -o lowerdir=/base,upperdir=/upper,workdir=/work /newroot
-rm -f /newroot/etc/resolv.conf
-touch /newroot/etc/resolv.conf
-cat /etc/resolv.conf > /newroot/etc/resolv.conf
-
-
-
-mkdir -p /newroot/old
-cd /newroot
-pivot_root . old
-
-mount -t proc proc /proc
-mount -t devtmpfs devtmpfs /dev
-mkdir -p /dev/pts
-mount -t devpts devpts /dev/pts
-mount -t proc proc /proc
-mount -t sysfs sysfs /sys
-
-
-
-mount
-
-touch /root_committed
-sudo -u runner /home/runner/work/openpilot/openpilot/selfdrive/test/build.sh
-ec=$?
-
-echo "hello"
-
-#sudo umount /newroot
-#sudo umount /base
-
-exit $ec

From 64ec9677110cae33c67dbb7b3fcf24b4f3e80e79 Mon Sep 17 00:00:00 2001
From: workinright <workinright@proton.me>
Date: Mon, 1 Sep 2025 08:01:32 +0200
Subject: [PATCH 3/9] test

---
 selfdrive/test/build.sh | 10 +++-------
 test                    |  0
 2 files changed, 3 insertions(+), 7 deletions(-)
 create mode 100644 test

diff --git a/selfdrive/test/build.sh b/selfdrive/test/build.sh
index 5ebea40ebe..af295eb04b 100755
--- a/selfdrive/test/build.sh
+++ b/selfdrive/test/build.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-#set -e
+set -e
 trap restore_root ERR
 ORG_PWD="$PWD"
 
@@ -21,6 +21,7 @@ commit_root() {
 
     sudo rm -rf /base /newroot /work
 
+    # finally, create the rootfs diff tarball (to be pushed into the CI native cache)
     mkdir -p /tmp/rootfs_cache
     sudo rm -f "$CACHE_ROOTFS_TARBALL_PATH" # remove the old tarball from previous run, if exists
     cd /upper
@@ -31,6 +32,7 @@ commit_root() {
 
     unpack_rootfs_tarball
 
+    # before the next tasks are run, finalize the environment for them
     prepare_mounts
 
     exit $ec
@@ -179,9 +181,3 @@ rm pyproject.toml uv.lock install_python_dependencies.sh
 
 # add a git safe directory for compiling openpilot
 sudo git config --global --add safe.directory /tmp/openpilot
-
-# finally, create the rootfs diff tarball (to be pushed into the CI native cache)
-
-
-# before the next tasks are run, finalize the environment for them
-#prepare_mounts
diff --git a/test b/test
new file mode 100644
index 0000000000..e69de29bb2

From 1e31ec7ab05ba9eee99288d86647d718998f4934 Mon Sep 17 00:00:00 2001
From: workinright <workinright@proton.me>
Date: Fri, 5 Sep 2025 21:01:33 +0200
Subject: [PATCH 4/9] commit

---
 selfdrive/test/build.sh        | 123 ++++++---------------------------
 selfdrive/test/build_common.sh | 113 ++++++++++++++++++++++++++++++
 2 files changed, 135 insertions(+), 101 deletions(-)
 create mode 100644 selfdrive/test/build_common.sh

diff --git a/selfdrive/test/build.sh b/selfdrive/test/build.sh
index af295eb04b..977d2f8f1d 100755
--- a/selfdrive/test/build.sh
+++ b/selfdrive/test/build.sh
@@ -1,114 +1,38 @@
 #!/usr/bin/env bash
 set -e
-trap restore_root ERR
 ORG_PWD="$PWD"
+SELF_PATH="$(realpath $0)"
+SCRIPT_DIR="$(dirname "$SELF_PATH")"
 
 REPO="$HOME/work/openpilot/openpilot"
 CACHE_ROOTFS_TARBALL_PATH="/tmp/rootfs_cache.tar"
 
-unpack_rootfs_tarball() {
-    cd /
-    sudo tar -xf "$CACHE_ROOTFS_TARBALL_PATH" 2>/dev/null || true
-    cd
-}
-
-commit_root() {
-    sudo mkdir -p /base /newroot /upper /work
-    
-    sudo unshare -f --kill-child -m $ORG_PWD/selfdrive/test/build.sh build_inside_namespace
-    ec=$?
-    echo "end of ns"
-
-    sudo rm -rf /base /newroot /work
-
-    # finally, create the rootfs diff tarball (to be pushed into the CI native cache)
-    mkdir -p /tmp/rootfs_cache
-    sudo rm -f "$CACHE_ROOTFS_TARBALL_PATH" # remove the old tarball from previous run, if exists
-    cd /upper
-    sudo tar -cf "$CACHE_ROOTFS_TARBALL_PATH" .
-    cd
-
-    sudo rm -rf /upper
-
-    unpack_rootfs_tarball
-
-    # before the next tasks are run, finalize the environment for them
-    prepare_mounts
-
-    exit $ec
-}
-
-prepare_mounts() {
-    # create and mount the required volumes where they're expected
-    mkdir -p /tmp/openpilot /tmp/scons_cache /tmp/comma_download_cache /tmp/openpilot_cache
-    sudo mount --bind "$REPO" /tmp/openpilot
-
-    sudo mount --bind "$REPO/.ci_cache/scons_cache" /tmp/scons_cache || true
-    sudo mount --bind "$REPO/.ci_cache/comma_download_cache" /tmp/comma_download_cache || true
-    sudo mount --bind "$REPO/.ci_cache/openpilot_cache" /tmp/openpilot_cache || true
-
-    # needed for the unit tests not to fail
-    sudo chmod 755 /sys/fs/pstore
-}
-
-restore_root() {
-    echo failed at ${BASH_LINENO[0]}
-}
-
-build_inside_namespace() {
-    mount --bind / /base
-    mount -t overlay overlay -o lowerdir=/base,upperdir=/upper,workdir=/work /newroot
-    rm -f /newroot/etc/resolv.conf
-    touch /newroot/etc/resolv.conf
-    cat /etc/resolv.conf > /newroot/etc/resolv.conf
-
-    mkdir -p /newroot/old
-    cd /newroot
-    pivot_root . old
-
-    mount -t proc proc /proc
-    mount -t devtmpfs devtmpfs /dev
-    mkdir -p /dev/pts
-    mount -t devpts devpts /dev/pts
-    mount -t proc proc /proc
-    mount -t sysfs sysfs /sys
-
-    touch /root_committed
-    sudo -u runner /home/runner/work/openpilot/openpilot/selfdrive/test/build.sh
-    ec=$?
-    exit $ec
-}
-
-if [ "$1" = "build_inside_namespace" ]
-then
-    build_inside_namespace
-    exit
-fi
+source "$SCRIPT_DIR/build_common.sh"
 
+# if the rootfs diff tarball (also created by this script) got restored from the CI native cache
 if [ -f "$CACHE_ROOTFS_TARBALL_PATH" ]
 then
-    # if the rootfs diff tarball (also created by this script) got restored from the CI native cache, unpack it, upgrading the rootfs
+    # apply it, upgrading the rootfs
     echo "restoring rootfs from the native build cache"
-    unpack_rootfs_tarball
+    apply_rootfs_diff
     rm "$CACHE_ROOTFS_TARBALL_PATH"
 
     # before the next tasks are run, finalize the environment for them
-    prepare_mounts
+    prepare_build
 
     # EXITS HERE - if the rootfs could been prepared entirely from the cache, there's no need for any further action like re-building
     exit 0
 else
     # otherwise, we'll have to install everything from scratch and build the tarball to be available for the next run
-    echo "no native build cache entry restored, rebuilding"
+    if ! [ -f /root_committed ]
+    then
+        echo "no native build cache entry restored, rebuilding"
+    fi
 fi
 
-# in order to be able to build a diff rootfs tarball, we need to commit its initial state by moving it on-the-fly to overlayfs;
-# below, we prepare the system and the new rootfs itself
-
-if ! [ -e /root_committed ]
-then
+# in order to be able to build a diff rootfs tarball, we need to commit its initial state
+# by moving it on-the-fly to overlayfs; below, we prepare the system and the new rootfs itself
 commit_root
-fi
 
 # -------- at this point, the original rootfs was committed and all the changes to it done below will be saved to the newly created rootfs diff tarball --------
 
@@ -118,11 +42,11 @@ DEBIAN_FRONTEND=noninteractive
 
 mkdir -p /tmp/tools
 cp "$REPO/tools/install_ubuntu_dependencies.sh" /tmp/tools/
-sudo /tmp/tools/install_ubuntu_dependencies.sh &>/dev/null
+sudo /tmp/tools/install_ubuntu_dependencies.sh
 
 sudo apt-get install -y --no-install-recommends \
     sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot python3-tk python3-dev \
-    apt-utils alien unzip tar curl xz-utils dbus gcc-arm-none-eabi tmux vim libx11-6 wget &>/dev/null
+    apt-utils alien unzip tar curl xz-utils dbus gcc-arm-none-eabi tmux vim libx11-6 wget
 
 sudo rm -rf /var/lib/apt/lists/*
 sudo apt-get clean
@@ -155,7 +79,6 @@ sudo ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so.2
 sudo mkdir -p /etc/ld.so.conf.d
 sudo bash -c "echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf"
 sudo ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf
-cd /
 rm -rf /tmp/opencl-driver-intel
 cd
 
@@ -166,18 +89,16 @@ NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
 QTWEBENGINE_DISABLE_SANDBOX=1
 
 # install and set up the Python dependencies needed
-sudo cp "/home/runner/work/openpilot/openpilot/pyproject.toml" "/home/runner/work/openpilot/openpilot/uv.lock" "/home/runner/work/openpilot/openpilot/tools/install_python_dependencies.sh" \
-    /home/runner/
+cp "$REPO/pyproject.toml" "$REPO/uv.lock" "$HOME/"
+mkdir "$HOME/tools"
+cp "$REPO/tools/install_python_dependencies.sh" "$HOME/tools/"
 
-cd
-rm -rf .venv
+VIRTUAL_ENV=/home/$USER/.venv
+PATH="$VIRTUAL_ENV/bin:$PATH"
 
-mkdir aaa
-cd aaa
-../install_python_dependencies.sh
 cd
-rm pyproject.toml uv.lock install_python_dependencies.sh
-
+tools/install_python_dependencies.sh
+rm -rf tools/ pyproject.toml uv.lock .cache
 
 # add a git safe directory for compiling openpilot
 sudo git config --global --add safe.directory /tmp/openpilot
diff --git a/selfdrive/test/build_common.sh b/selfdrive/test/build_common.sh
new file mode 100644
index 0000000000..a173288222
--- /dev/null
+++ b/selfdrive/test/build_common.sh
@@ -0,0 +1,113 @@
+error_handler() {
+    echo failed at ${BASH_LINENO[0]}
+}
+trap error_handler ERR
+
+create_rootfs_diff() {
+    sudo rm -f "$CACHE_ROOTFS_TARBALL_PATH" # remove the old diff tarball from previous run, if exists
+    cd /upper
+    sudo tar -cf "$CACHE_ROOTFS_TARBALL_PATH" .
+    cd
+}
+
+apply_rootfs_diff() {
+    cd /
+    sudo tar -xf "$CACHE_ROOTFS_TARBALL_PATH" 2>/dev/null || true
+    cd
+}
+
+prepare_build() {
+    # create and mount the required volumes where they're expected
+    mkdir -p /tmp/openpilot /tmp/scons_cache /tmp/comma_download_cache /tmp/openpilot_cache
+    sudo mount --bind "$REPO" /tmp/openpilot
+
+    sudo mount --bind "$REPO/.ci_cache/scons_cache" /tmp/scons_cache || true
+    sudo mount --bind "$REPO/.ci_cache/comma_download_cache" /tmp/comma_download_cache || true
+    sudo mount --bind "$REPO/.ci_cache/openpilot_cache" /tmp/openpilot_cache || true
+
+    # needed for the unit tests not to fail
+    sudo chmod 755 /sys/fs/pstore
+}
+
+post_commit_root() {
+    # we have the diff tarball, now let's remove the folder too
+    sudo rm -rf /upper
+
+    # now we apply it straight away
+    apply_rootfs_diff
+
+    # before the next tasks are run, finalize the environment for them
+    prepare_build
+}
+
+# warning: this function initiates a somewhat complicated program flow, follow carefully
+# (even despite this part was made sure to not be too relevant for the rest of the job)
+commit_root() {
+    # if that's a first execution
+    if ! [ -e /root_committed ]
+    then
+        # prepare directories
+        sudo mkdir -p /base /newroot /upper /work
+        
+        # re-execute the main script (causing it to go straight to `build_inside_namespace`), but
+        # inside the newly created namespace, in a way which would cause all mounts
+        # created to automatically umount before it exits
+        sudo unshare -f --kill-child -m "$SELF_PATH" build_inside_namespace
+        ec=$?
+
+        # after it exited, remove the created directories (except the one containing created diff)
+        sudo rm -rf /base /newroot /work
+
+        # finally, create the rootfs diff tarball (to be pushed into the CI native cache)
+        create_rootfs_diff
+
+        # after creating the rootfs diff, bring the system into a state as if it was restored from cache
+        post_commit_root
+
+        exit $ec
+    fi
+}
+
+reexecute() {
+    touch /root_committed
+    sudo -u runner "$SELF_PATH"
+    ec=$?
+    exit $ec
+}
+
+# that's where the script goes after being re-executed for the first time
+build_inside_namespace() {
+    # initialize the mounts namespace on overlayfs to be able to prepare the rootfs diff
+    mount --bind / /base
+    mount -t overlay overlay -o lowerdir=/base,upperdir=/upper,workdir=/work /newroot
+
+    # apply the current DNS config (beware: systemd often symlinks /etc/resolv.conf, that's why it's needed)
+    rm -f /newroot/etc/resolv.conf
+    touch /newroot/etc/resolv.conf
+    cat /etc/resolv.conf > /newroot/etc/resolv.conf
+
+    # switch the namespace's root mount to the newly created one
+    mkdir -p /newroot/old
+    cd /newroot
+    pivot_root . old
+
+    # initialize basic required POSIX-standard additional mounts
+    mount -t proc proc /proc
+    mount -t devtmpfs devtmpfs /dev
+    mkdir -p /dev/pts
+    mount -t devpts devpts /dev/pts
+    mount -t proc proc /proc
+    mount -t sysfs sysfs /sys
+
+    # re-execute the main script for the 2nd time, causing it to go back to the main flow
+    # (but this time already inside the newly created namespace)
+    reexecute
+
+    # after the main flow terminates and the namespace exist, post_commit_root is executed - be sure to look at it
+}
+
+if [ "$1" = "build_inside_namespace" ]
+then
+    build_inside_namespace
+    exit
+fi

From 99df8f58728a6dcc617947f612ec0ffd4c3d1a5e Mon Sep 17 00:00:00 2001
From: workinright <workinright@proton.me>
Date: Sat, 6 Sep 2025 16:42:31 +0200
Subject: [PATCH 5/9] commit

---
 .github/workflows/setup/action.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/setup/action.yaml b/.github/workflows/setup/action.yaml
index a21616dab1..1711dfefdd 100644
--- a/.github/workflows/setup/action.yaml
+++ b/.github/workflows/setup/action.yaml
@@ -30,7 +30,7 @@ runs:
     - shell: bash
       run: git lfs pull
 
-    # build scons cache
+    # restore scons cache
     - id: date
       shell: bash
       run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV ; echo "CACHE_FILE_HASHES=$(sha256sum selfdrive/test/build.sh tools/install_ubuntu_dependencies.sh tools/install_python_dependencies.sh | cut -d" " -f1 | tr '\n' '_')" >> $GITHUB_ENV
@@ -45,7 +45,7 @@ runs:
           scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}
           scons-${{ runner.arch }}
 
-    # try to restore rootfs image cache
+    # restore rootfs image cache
     - id: rootfs-cache
       uses: actions/cache/restore@v4
       with:

From 6fb3d6941facc11b2c496e759620cc5614e4c37d Mon Sep 17 00:00:00 2001
From: workinright <workinright@proton.me>
Date: Sat, 6 Sep 2025 16:43:48 +0200
Subject: [PATCH 6/9] commit

---
 .github/workflows/setup/action.yaml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/setup/action.yaml b/.github/workflows/setup/action.yaml
index 1711dfefdd..fa925c7f85 100644
--- a/.github/workflows/setup/action.yaml
+++ b/.github/workflows/setup/action.yaml
@@ -47,17 +47,19 @@ runs:
 
     # restore rootfs image cache
     - id: rootfs-cache
-      uses: actions/cache/restore@v4
+      uses: actions/download-artifact@v4
       with:
         path: /tmp/rootfs_cache.tar
-        key: rootfs-${{ runner.arch }}-${{ env.CACHE_FILE_HASHES }}
+        name: rootfs-${{ runner.arch }}-${{ env.CACHE_FILE_HASHES }}
+      continue-on-error: true
     # build our rootfs image
     - shell: bash
       run: eval ${{ env.BUILD }}
     # try to save rootfs image cache
     - name: Save rootfs cache
-      uses: actions/cache/save@v4
+      uses: actions/upload-artifact@v4
       #if: github.ref == 'refs/heads/master'
       with:
         path: /tmp/rootfs_cache.tar
-        key: rootfs-${{ runner.arch }}-${{ env.CACHE_FILE_HASHES }}
+        name: rootfs-${{ runner.arch }}-${{ env.CACHE_FILE_HASHES }}
+      continue-on-error: true

From ae29289f3c35f2f64f4f7c16b449322fa28952c2 Mon Sep 17 00:00:00 2001
From: workinright <workinright@proton.me>
Date: Sat, 6 Sep 2025 18:05:51 +0200
Subject: [PATCH 7/9] commit

---
 .github/workflows/selfdrive_tests.yaml    | 11 ++++++++
 .github/workflows/setup-cache/action.yaml | 32 +++++++++++++++++++++++
 .github/workflows/setup/action.yaml       |  9 +------
 3 files changed, 44 insertions(+), 8 deletions(-)
 create mode 100644 .github/workflows/setup-cache/action.yaml

diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml
index 44ff010862..710d75e410 100644
--- a/.github/workflows/selfdrive_tests.yaml
+++ b/.github/workflows/selfdrive_tests.yaml
@@ -88,6 +88,17 @@ jobs:
     - uses: ./.github/workflows/compile-openpilot
       timeout-minutes: 30
 
+  setup-cache:
+    name: setup cache
+    runs-on: ubuntu-24.04
+    needs: ["build"]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: setup-cache
+        uses: ./.github/workflows/setup-cache
+
   build_mac:
     name: build macOS
     if: false  # temp disable since homebrew install is getting stuck
diff --git a/.github/workflows/setup-cache/action.yaml b/.github/workflows/setup-cache/action.yaml
new file mode 100644
index 0000000000..bd073c361e
--- /dev/null
+++ b/.github/workflows/setup-cache/action.yaml
@@ -0,0 +1,32 @@
+name: 'setup-cache'
+
+inputs:
+  docker_hub_pat:
+    description: 'Auth token for Docker Hub, required for BuildJet jobs'
+    required: false
+    default: ''
+  sleep_time:
+    description: 'Time to sleep between retries'
+    required: false
+    default: 30
+
+outputs:
+  duration:
+    description: 'Duration of the setup process in seconds'
+    value: ${{ steps.get_duration.outputs.duration }}
+
+runs:
+  using: "composite"
+  steps:
+    # build cache
+    - id: date
+      shell: bash
+      run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV ; echo "CACHE_FILE_HASHES=$(sha256sum selfdrive/test/build.sh tools/install_ubuntu_dependencies.sh tools/install_python_dependencies.sh | cut -d" " -f1 | tr '\n' '_')" >> $GITHUB_ENV
+    - shell: bash
+      run: echo "$CACHE_COMMIT_DATE"
+    - name: Save rootfs cache
+      uses: actions/upload-artifact@v4
+      #if: github.ref == 'refs/heads/master'
+      with:
+        path: /tmp/rootfs_cache.tar
+        name: rootfs-${{ runner.arch }}-${{ env.CACHE_FILE_HASHES }}
diff --git a/.github/workflows/setup/action.yaml b/.github/workflows/setup/action.yaml
index fa925c7f85..f3d320de6a 100644
--- a/.github/workflows/setup/action.yaml
+++ b/.github/workflows/setup/action.yaml
@@ -55,11 +55,4 @@ runs:
     # build our rootfs image
     - shell: bash
       run: eval ${{ env.BUILD }}
-    # try to save rootfs image cache
-    - name: Save rootfs cache
-      uses: actions/upload-artifact@v4
-      #if: github.ref == 'refs/heads/master'
-      with:
-        path: /tmp/rootfs_cache.tar
-        name: rootfs-${{ runner.arch }}-${{ env.CACHE_FILE_HASHES }}
-      continue-on-error: true
+

From bbf3c4d5c5c71d36c7874d9b2d3355804f027439 Mon Sep 17 00:00:00 2001
From: workinright <workinright@proton.me>
Date: Sat, 6 Sep 2025 18:13:28 +0200
Subject: [PATCH 8/9] commit

---
 .github/workflows/selfdrive_tests.yaml    | 11 --------
 .github/workflows/setup-cache/action.yaml | 32 -----------------------
 .github/workflows/setup/action.yaml       |  8 +++++-
 3 files changed, 7 insertions(+), 44 deletions(-)
 delete mode 100644 .github/workflows/setup-cache/action.yaml

diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml
index 710d75e410..44ff010862 100644
--- a/.github/workflows/selfdrive_tests.yaml
+++ b/.github/workflows/selfdrive_tests.yaml
@@ -88,17 +88,6 @@ jobs:
     - uses: ./.github/workflows/compile-openpilot
       timeout-minutes: 30
 
-  setup-cache:
-    name: setup cache
-    runs-on: ubuntu-24.04
-    needs: ["build"]
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-      - name: setup-cache
-        uses: ./.github/workflows/setup-cache
-
   build_mac:
     name: build macOS
     if: false  # temp disable since homebrew install is getting stuck
diff --git a/.github/workflows/setup-cache/action.yaml b/.github/workflows/setup-cache/action.yaml
deleted file mode 100644
index bd073c361e..0000000000
--- a/.github/workflows/setup-cache/action.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-name: 'setup-cache'
-
-inputs:
-  docker_hub_pat:
-    description: 'Auth token for Docker Hub, required for BuildJet jobs'
-    required: false
-    default: ''
-  sleep_time:
-    description: 'Time to sleep between retries'
-    required: false
-    default: 30
-
-outputs:
-  duration:
-    description: 'Duration of the setup process in seconds'
-    value: ${{ steps.get_duration.outputs.duration }}
-
-runs:
-  using: "composite"
-  steps:
-    # build cache
-    - id: date
-      shell: bash
-      run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV ; echo "CACHE_FILE_HASHES=$(sha256sum selfdrive/test/build.sh tools/install_ubuntu_dependencies.sh tools/install_python_dependencies.sh | cut -d" " -f1 | tr '\n' '_')" >> $GITHUB_ENV
-    - shell: bash
-      run: echo "$CACHE_COMMIT_DATE"
-    - name: Save rootfs cache
-      uses: actions/upload-artifact@v4
-      #if: github.ref == 'refs/heads/master'
-      with:
-        path: /tmp/rootfs_cache.tar
-        name: rootfs-${{ runner.arch }}-${{ env.CACHE_FILE_HASHES }}
diff --git a/.github/workflows/setup/action.yaml b/.github/workflows/setup/action.yaml
index f3d320de6a..449fac92f4 100644
--- a/.github/workflows/setup/action.yaml
+++ b/.github/workflows/setup/action.yaml
@@ -55,4 +55,10 @@ runs:
     # build our rootfs image
     - shell: bash
       run: eval ${{ env.BUILD }}
-
+    # try to save rootfs image cache
+    - name: Save rootfs cache
+      uses: actions/upload-artifact@v4
+      #if: github.ref == 'refs/heads/master'
+      with:
+        path: /tmp/rootfs_cache.tar
+        name: rootfs-${{ runner.arch }}-${{ env.CACHE_FILE_HASHES }}

From bdfecd594558ca06b840ecf08c10ec788cc3c052 Mon Sep 17 00:00:00 2001
From: workinright <workinright@proton.me>
Date: Sat, 6 Sep 2025 18:20:36 +0200
Subject: [PATCH 9/9] commit

---
 .github/workflows/setup/action.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/setup/action.yaml b/.github/workflows/setup/action.yaml
index 449fac92f4..28016f288e 100644
--- a/.github/workflows/setup/action.yaml
+++ b/.github/workflows/setup/action.yaml
@@ -61,4 +61,4 @@ runs:
       #if: github.ref == 'refs/heads/master'
       with:
         path: /tmp/rootfs_cache.tar
-        name: rootfs-${{ runner.arch }}-${{ env.CACHE_FILE_HASHES }}
+        name: rootfs-${{ runner.arch }}-${{ env.CACHE_FILE_HASHES }}-${{ github.run_id }}