diff --git a/.github/workflows/prebuilt.yaml b/.github/workflows/prebuilt.yaml index 3b88011f5e..8b16ea90b9 100644 --- a/.github/workflows/prebuilt.yaml +++ b/.github/workflows/prebuilt.yaml @@ -5,12 +5,8 @@ on: workflow_dispatch: env: - BASE_IMAGE: openpilot-base - DOCKER_REGISTRY: ghcr.io/commaai - DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} - BUILD: | - DOCKER_BUILDKIT=1 docker build --pull --cache-to type=inline --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base . + BUILD: selfdrive/test/docker_build.sh prebuilt jobs: build_prebuilt: @@ -18,7 +14,7 @@ jobs: runs-on: ubuntu-20.04 if: github.repository == 'commaai/openpilot' env: - IMAGE_NAME: openpilot-prebuilt + PUSH_IMAGE: true steps: - name: Wait for green check mark if: ${{ github.event_name != 'workflow_dispatch' }} @@ -31,13 +27,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - name: Build Docker image - run: | - eval "$BUILD" - DOCKER_BUILDKIT=1 docker build --pull --cache-to type=inline --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f Dockerfile.openpilot . - - name: Push to container registry + - name: Build and Push docker image run: | $DOCKER_LOGIN - docker push $DOCKER_REGISTRY/$IMAGE_NAME:latest - docker tag $DOCKER_REGISTRY/$IMAGE_NAME:latest $DOCKER_REGISTRY/$IMAGE_NAME:$GITHUB_SHA - docker push $DOCKER_REGISTRY/$IMAGE_NAME:$GITHUB_SHA + eval "$BUILD" diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 61d1602a4c..64c96280f8 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -12,20 +12,17 @@ concurrency: env: PYTHONWARNINGS: error - BASE_IMAGE: openpilot-base CL_BASE_IMAGE: openpilot-base-cl - DOCKER_REGISTRY: ghcr.io/commaai AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }} DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} - BUILD: | - DOCKER_BUILDKIT=1 docker build --pull --cache-to type=inline --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base . + BUILD: selfdrive/test/docker_build.sh base RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v ~/scons_cache:/tmp/scons_cache -v ~/comma_download_cache:/tmp/comma_download_cache -v ~/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c - BUILD_CL: | - DOCKER_BUILDKIT=1 docker build --cache-to type=inline --cache-from $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $CL_BASE_IMAGE:latest -f Dockerfile.openpilot_base_cl . + BUILD_CL: selfdrive/test/docker_build.sh cl + RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v ~/scons_cache:/tmp/scons_cache -v ~/comma_download_cache:/tmp/comma_download_cache -v ~/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c UNIT_TEST: coverage run --append -m unittest discover @@ -190,6 +187,8 @@ jobs: name: docker push runs-on: ubuntu-20.04 if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' + env: + PUSH_IMAGE: true steps: - uses: actions/checkout@v3 with: @@ -197,20 +196,10 @@ jobs: - uses: ./.github/workflows/setup with: git-lfs: false - - name: Push to container registry - run: | - $DOCKER_LOGIN - docker push $DOCKER_REGISTRY/$BASE_IMAGE:latest - docker tag $DOCKER_REGISTRY/$BASE_IMAGE:latest $DOCKER_REGISTRY/$BASE_IMAGE:$GITHUB_SHA - docker push $DOCKER_REGISTRY/$BASE_IMAGE:$GITHUB_SHA - - name: Build CL Docker image - run: eval "$BUILD_CL" - - name: Push to container registry + - name: Build and push CL Docker image run: | $DOCKER_LOGIN - docker push $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest - docker tag $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest $DOCKER_REGISTRY/$CL_BASE_IMAGE:$GITHUB_SHA - docker push $DOCKER_REGISTRY/$CL_BASE_IMAGE:$GITHUB_SHA + eval "$BUILD_CL" static_analysis: name: static analysis diff --git a/.github/workflows/tools_tests.yaml b/.github/workflows/tools_tests.yaml index f1f4868dcb..470c7662a4 100644 --- a/.github/workflows/tools_tests.yaml +++ b/.github/workflows/tools_tests.yaml @@ -13,16 +13,14 @@ concurrency: env: BASE_IMAGE: openpilot-base CL_BASE_IMAGE: openpilot-base-cl - DOCKER_REGISTRY: ghcr.io/commaai DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} - BUILD: | - DOCKER_BUILDKIT=1 docker build --pull --cache-to type=inline --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base . + BUILD: selfdrive/test/docker_build.sh base RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v ~/scons_cache:/tmp/scons_cache -v ~/comma_download_cache:/tmp/comma_download_cache -v ~/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c - BUILD_CL: | - DOCKER_BUILDKIT=1 docker build --pull --cache-to type=inline --cache-from $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $CL_BASE_IMAGE:latest -f Dockerfile.openpilot_base_cl . + BUILD_CL: selfdrive/test/docker_build.sh cl + RUN_CL: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v ~/scons_cache:/tmp/scons_cache -v ~/comma_download_cache:/tmp/comma_download_cache -v ~/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c @@ -49,8 +47,6 @@ jobs: simulator: name: simulator runs-on: ubuntu-20.04 - env: - IMAGE_NAME: openpilot-sim if: github.repository == 'commaai/openpilot' timeout-minutes: 45 steps: @@ -62,14 +58,14 @@ jobs: setup_docker_scons_cache: true - name: Build base cl image run: eval "$BUILD_CL" - - name: Build simulator image - run: | - DOCKER_BUILDKIT=1 docker build --cache-to type=inline --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/sim/Dockerfile.sim . - - name: Push to container registry + - name: Setup to push to repo if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot' - run: | + run: + export PUSH_IMAGE=true $DOCKER_LOGIN - docker push $DOCKER_REGISTRY/$IMAGE_NAME:latest + - name: Build and push sim image + run: | + selfdrive/test/docker_build.sh sim docs: name: build docs @@ -83,14 +79,14 @@ jobs: with: setup_docker_scons_cache: true git_lfs: false - - name: Build docs image - run: | - DOCKER_BUILDKIT=1 docker build --cache-to type=inline --cache-from $DOCKER_REGISTRY/openpilot-docs:latest -t $DOCKER_REGISTRY/openpilot-docs:latest -f docs/docker/Dockerfile . - - name: Push docker container + - name: Setup to push to repo if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' - run: | + run: + export PUSH_IMAGE=true $DOCKER_LOGIN - docker push $DOCKER_REGISTRY/openpilot-docs:latest + - name: Build and push docs image + run: | + selfdrive/test/docker_build.sh docs devcontainer: name: devcontainer diff --git a/selfdrive/test/docker_build.sh b/selfdrive/test/docker_build.sh new file mode 100755 index 0000000000..b18c48123a --- /dev/null +++ b/selfdrive/test/docker_build.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -e + +if [ $1 = "base" ]; then + export DOCKER_IMAGE=openpilot-base + export DOCKER_FILE=Dockerfile.openpilot_base +elif [ $1 = "docs" ]; then + export DOCKER_IMAGE=openpilot-docs + export DOCKER_FILE=docs/docker/Dockerfile +elif [ $1 = "sim" ]; then + export DOCKER_IMAGE=openpilot-sim + export DOCKER_FILE=tools/sim/Dockerfile.sim +elif [ $1 = "prebuilt" ]; then + export DOCKER_IMAGE=openpilot-prebuilt + export DOCKER_FILE=Dockerfile.openpilot +elif [ $1 = "cl" ]; then + export DOCKER_IMAGE=openpilot-base-cl + export DOCKER_FILE=Dockerfile.openpilot_base_cl +else + echo "Invalid docker build image $1" + exit 1 +fi + +export DOCKER_REGISTRY=ghcr.io/commaai +export COMMIT_SHA=$(git rev-parse HEAD) + +LOCAL_TAG=$DOCKER_IMAGE +REMOTE_TAG=$DOCKER_REGISTRY/$LOCAL_TAG +REMOTE_SHA_TAG=$REMOTE_TAG:$COMMIT_SHA + +SCRIPT_DIR=$(dirname "$0") +OPENPILOT_DIR=$SCRIPT_DIR/../../ + +DOCKER_BUILDKIT=1 docker build --cache-to type=inline --cache-from type=registry,ref=$REMOTE_TAG -t $REMOTE_TAG -t $LOCAL_TAG -f $OPENPILOT_DIR/$DOCKER_FILE $OPENPILOT_DIR + +if [[ ! -z "$PUSH_IMAGE" ]]; +then + docker push $REMOTE_TAG + docker tag $REMOTE_TAG $REMOTE_SHA_TAG + docker push $REMOTE_SHA_TAG +fi \ No newline at end of file