diff --git a/.github/workflows/setup/action.yaml b/.github/workflows/setup/action.yaml
index 495c4745ee..8bb1ccc376 100644
--- a/.github/workflows/setup/action.yaml
+++ b/.github/workflows/setup/action.yaml
@@ -68,7 +68,8 @@ runs:
         find . -type f -executable -not -perm 755 -exec chmod 755 {} \;
         find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \;
     - id: setup-buildx-action
-      name: setup docker buildx for registry cache
+      if: contains(runner.name, 'buildjet')
+      name: Set up Docker Buildx on buildjet to ensure a consistent cache
       uses: docker/setup-buildx-action@v2
       with:
         driver: docker-container
diff --git a/selfdrive/test/docker_build.sh b/selfdrive/test/docker_build.sh
index d5a05fecad..e0ba54f058 100755
--- a/selfdrive/test/docker_build.sh
+++ b/selfdrive/test/docker_build.sh
@@ -17,15 +17,7 @@ fi
 
 source $SCRIPT_DIR/docker_common.sh $1 "$TAG_SUFFIX"
 
-if [ -n "$PUSH_IMAGE" ]; then
-  REG_CACHE="--cache-to type=registry,ref=$REMOTE_CACHE_TAG,mode=max"
-fi
-
-DOCKER_BUILDKIT=1 docker buildx build --platform $PLATFORM --load \
-  --cache-from type=registry,ref=$REMOTE_CACHE_TAG \
-  --cache-from type=registry,ref=$REMOTE_TAG \
-  $REG_CACHE \
-  -t $REMOTE_TAG -t $LOCAL_TAG -f $OPENPILOT_DIR/$DOCKER_FILE $OPENPILOT_DIR
+DOCKER_BUILDKIT=1 docker buildx build --platform $PLATFORM --load --cache-to type=inline --cache-from type=registry,ref=$REMOTE_TAG -t $REMOTE_TAG -t $LOCAL_TAG -f $OPENPILOT_DIR/$DOCKER_FILE $OPENPILOT_DIR
 
 if [ -n "$PUSH_IMAGE" ]; then
   docker push $REMOTE_TAG
diff --git a/selfdrive/test/docker_common.sh b/selfdrive/test/docker_common.sh
index f1422bf643..92da71ba66 100644
--- a/selfdrive/test/docker_common.sh
+++ b/selfdrive/test/docker_common.sh
@@ -22,4 +22,3 @@ TAG_SUFFIX=$2
 LOCAL_TAG=$DOCKER_IMAGE$TAG_SUFFIX
 REMOTE_TAG=$DOCKER_REGISTRY/$LOCAL_TAG
 REMOTE_SHA_TAG=$DOCKER_REGISTRY/$LOCAL_TAG:$COMMIT_SHA
-REMOTE_CACHE_TAG=$DOCKER_REGISTRY/$LOCAL_TAG:cache