From 71e76c3d0fa819b8aa9d3580c96b52480febf3d4 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 21 Aug 2022 19:14:36 -0700 Subject: [PATCH] CI: Actions cleanup + speedup (#25514) * actions cache cleanup * release build cleanup * fetch dpeth --- .github/workflows/badges.yaml | 25 +-- .github/workflows/selfdrive_tests.yaml | 230 +++++-------------------- .github/workflows/setup/action.yaml | 45 +++++ .github/workflows/tools_tests.yaml | 40 ----- 4 files changed, 88 insertions(+), 252 deletions(-) create mode 100644 .github/workflows/setup/action.yaml diff --git a/.github/workflows/badges.yaml b/.github/workflows/badges.yaml index 223c734863..644745920c 100644 --- a/.github/workflows/badges.yaml +++ b/.github/workflows/badges.yaml @@ -7,12 +7,7 @@ on: env: BASE_IMAGE: openpilot-base DOCKER_REGISTRY: ghcr.io/commaai - - BUILD: | - docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) || true - docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true - docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base . - RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c + RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/sh -c jobs: badges: @@ -23,23 +18,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - - name: Cache scons - id: scons-cache - # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. - uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b - env: - CACHE_SKIP_SAVE: true - with: - path: /tmp/scons_cache - key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- - restore-keys: | - scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- - scons- - - - name: Build Docker image - run: eval "$BUILD" - + - uses: ./.github/workflows/setup - name: Push badges run: | ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/ui/translations/create_badges.py" diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 7397e30f83..8529da8e89 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -17,7 +17,7 @@ env: docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base . - RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c + RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c BUILD_CL: | docker pull $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest || true @@ -27,11 +27,10 @@ env: UNIT_TEST: coverage run --append -m unittest discover jobs: - # TODO: once actions/cache supports read only mode, use the cache for all jobs build_release: name: build release runs-on: ubuntu-20.04 - timeout-minutes: 50 + timeout-minutes: 30 env: STRIPPED_DIR: /tmp/releasepilot steps: @@ -39,44 +38,29 @@ jobs: with: fetch-depth: 0 submodules: true - - name: Pull LFS - run: git lfs pull + - name: Build devel + run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh + - uses: ./.github/workflows/setup - name: Check submodules if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot' run: release/check-submodules.sh - - name: Cache scons - id: scons-cache - # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. - uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b - env: - CACHE_SKIP_SAVE: true - with: - path: /tmp/scons_cache - key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- - restore-keys: | - scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- - scons- - - name: Build devel + - name: Build openpilot and run checks + run: | + cd $STRIPPED_DIR + ${{ env.RUN }} "CI=1 python selfdrive/manager/build.py" + - name: Run tests run: | - TARGET_DIR=$STRIPPED_DIR release/build_devel.sh - cp Dockerfile.openpilot_base $STRIPPED_DIR + cd $STRIPPED_DIR + ${{ env.RUN }} "release/check-dirty.sh && \ + python -m unittest discover selfdrive/car" + - name: pre-commit + run: | + cd $GITHUB_WORKSPACE cp .pre-commit-config.yaml $STRIPPED_DIR cp .pylintrc $STRIPPED_DIR cp mypy.ini $STRIPPED_DIR - - name: Build Docker image - run: | - eval "$BUILD" - rm $STRIPPED_DIR/Dockerfile.openpilot_base - - name: Build openpilot and run checks - run: | - cd $STRIPPED_DIR - ${{ env.RUN }} "CI=1 python selfdrive/manager/build.py && \ - pre-commit run --all && \ - rm .pre-commit-config.yaml && \ - rm .pylintrc && \ - rm mypy.ini && \ - release/check-dirty.sh && \ - python -m unittest discover selfdrive/car" + cd $STRIPPED_DIR + ${{ env.RUN }} "pre-commit run --all" build_all: name: build all @@ -86,26 +70,14 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - name: Cache scons - id: scons-cache - # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. - uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b - env: - CACHE_SKIP_SAVE: ${{ github.ref != 'refs/heads/master' || github.repository != 'commaai/openpilot' }} + - uses: ./.github/workflows/setup with: - path: /tmp/scons_cache - key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-${{ steps.date.outputs.time }} - restore-keys: | - scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- - scons- - - name: Build Docker image - run: eval "$BUILD" + save-cache: true - name: Build openpilot with all flags run: ${{ env.RUN }} "scons -j$(nproc) --extras --test && release/check-dirty.sh" - name: Cleanup scons cache run: | - ${{ env.RUN }} "scons -j$(nproc) --extras --test && \ - rm -rf /tmp/scons_cache/* && \ + ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \ scons -j$(nproc) --extras --test --cache-populate" #build_mac: @@ -175,21 +147,9 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - name: Cache scons - id: scons-cache - # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. - uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b - env: - CACHE_SKIP_SAVE: true - with: - path: /tmp/scons_cache - key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- - restore-keys: | - scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- - scons- + - uses: ./.github/workflows/setup - name: Build Docker image run: | - eval "$BUILD" docker pull $DOCKER_REGISTRY/$IMAGE_NAME:latest || true docker build --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/webcam/Dockerfile . - name: Build openpilot @@ -244,30 +204,11 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - name: Cache dependencies - id: dependency-cache - uses: actions/cache@v2 - with: - path: /tmp/comma_download_cache - key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/test_valgrind_replay.py') }} - - name: Cache scons - id: scons-cache - # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. - uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b - env: - CACHE_SKIP_SAVE: true - with: - path: /tmp/scons_cache - key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- - restore-keys: | - scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- - scons- - - name: Build Docker image - run: eval "$BUILD" + - uses: ./.github/workflows/setup - name: Run valgrind run: | ${{ env.RUN }} "scons -j$(nproc) && \ - FILEREADER_CACHE=1 python selfdrive/test/test_valgrind_replay.py" + python selfdrive/test/test_valgrind_replay.py" - name: Print logs if: always() run: cat selfdrive/test/valgrind_logs.txt @@ -277,28 +218,10 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 50 steps: - - name: Get current date - id: date - run: echo "::set-output name=time::$(date +'%s')" - - name: Output timestamp - run: echo $TIMESTAMP - env: - TIMESTAMP: ${{ steps.date.outputs.time }} - uses: actions/checkout@v3 with: submodules: true - - name: Cache scons - id: scons-cache - # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. - uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b - env: - CACHE_SKIP_SAVE: ${{ github.ref != 'refs/heads/master' || github.repository != 'commaai/openpilot' }} - with: - path: /tmp/scons_cache - key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-${{ steps.date.outputs.time }} - restore-keys: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- - - name: Build Docker image - run: eval "$BUILD" + - uses: ./.github/workflows/setup - name: Run unit tests run: | ${{ env.RUN }} "export SKIP_LONG_TESTS=1 && \ @@ -339,30 +262,17 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - name: Cache dependencies + - uses: ./.github/workflows/setup + - name: Cache test routes id: dependency-cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: /tmp/comma_download_cache - key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/test_processes.py') }} - - name: Cache scons - id: scons-cache - # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. - uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b - env: - CACHE_SKIP_SAVE: true - with: - path: /tmp/scons_cache - key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- - restore-keys: | - scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- - scons- - - name: Build Docker image - run: eval "$BUILD" + key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit') }} - name: Run replay run: | ${{ env.RUN }} "scons -j$(nproc) && \ - FILEREADER_CACHE=1 CI=1 coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \ + CI=1 coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \ coverage xml" - name: Print diff if: always() @@ -389,35 +299,16 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - name: Pull LFS - run: git lfs pull - - name: Cache dependencies - id: dependency-cache - uses: actions/cache@v2 - with: - path: /tmp/comma_download_cache - key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/model_replay.py') }} - - name: Cache scons - id: scons-cache - # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. - uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b - env: - CACHE_SKIP_SAVE: true - with: - path: /tmp/scons_cache - key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- - restore-keys: | - scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- - scons- + - uses: ./.github/workflows/setup - name: Build Docker image - # Sim docker is needed to get the intel OPENCL drivers + # Sim docker is needed to get the OpenCL drivers run: eval "$BUILD_CL" - name: Run replay run: | ${{ env.RUN_CL }} "scons -j$(nproc) && \ - ONNXCPU=1 FILEREADER_CACHE=1 CI=1 coverage run \ - selfdrive/test/process_replay/model_replay.py -j$(nproc) && \ - coverage xml" + ONNXCPU=1 CI=1 coverage run \ + selfdrive/test/process_replay/model_replay.py -j$(nproc) && \ + coverage xml" test_longitudinal: name: longitudinal @@ -427,20 +318,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - name: Cache scons - id: scons-cache - # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. - uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b - env: - CACHE_SKIP_SAVE: true - with: - path: /tmp/scons_cache - key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- - restore-keys: | - scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- - scons- - - name: Build Docker image - run: eval "$BUILD" + - uses: ./.github/workflows/setup - name: Test longitudinal run: | ${{ env.RUN }} "mkdir -p selfdrive/test/out && \ @@ -469,30 +347,17 @@ jobs: - uses: actions/checkout@v3 with: submodules: true - - name: Cache dependencies + - uses: ./.github/workflows/setup + - name: Cache test routes id: dependency-cache - uses: actions/cache@v2 + uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b with: path: /tmp/comma_download_cache key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }} - - name: Cache scons - id: scons-cache - # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. - uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b - env: - CACHE_SKIP_SAVE: true - with: - path: /tmp/scons_cache - key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- - restore-keys: | - scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- - scons- - - name: Build Docker image - run: eval "$BUILD" - name: Test car models run: | ${{ env.RUN }} "scons -j$(nproc) --test && \ - FILEREADER_CACHE=1 coverage run -m pytest selfdrive/car/tests/test_models.py && \ + coverage run -m pytest selfdrive/car/tests/test_models.py && \ coverage xml && \ chmod -R 777 /tmp/comma_download_cache" env: @@ -530,20 +395,7 @@ jobs: with: submodules: true ref: ${{ github.event.pull_request.base.ref }} - - name: Cache scons - id: scons-cache - # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. - uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b - env: - CACHE_SKIP_SAVE: true - with: - path: /tmp/scons_cache - key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- - restore-keys: | - scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- - scons- - - name: Build Docker image - run: eval "$BUILD" + - uses: ./.github/workflows/setup - name: Get base car info run: | ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_info.py --path /tmp/openpilot_cache/base_car_info" diff --git a/.github/workflows/setup/action.yaml b/.github/workflows/setup/action.yaml new file mode 100644 index 0000000000..79ec921235 --- /dev/null +++ b/.github/workflows/setup/action.yaml @@ -0,0 +1,45 @@ +name: 'openpilot env setup' + +env: + BASE_IMAGE: openpilot-base + DOCKER_REGISTRY: ghcr.io/commaai + BUILD: | + docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) || true + docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true + docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base . + +inputs: + save-cache: + default: false + required: false + +runs: + using: "composite" + steps: + # do this after checkout to ensure our custom LFS config is used to pull from GitLab + - shell: bash + run: git lfs pull + + # build cache + - id: date + shell: bash + run: echo "::set-output name=date::$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d')" + - shell: bash + run: echo "${{ steps.date.outputs.date }}" + - shell: bash + run: echo "CACHE_SKIP_SAVE=true" >> $GITHUB_ENV + if: github.ref != 'refs/heads/master' || inputs.save-cache == 'false' + - id: scons-cache + # TODO: change the version to the released version + # when https://github.com/actions/cache/pull/489 (or 571) is merged. + uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b + with: + path: /tmp/scons_cache + key: scons-${{ steps.date.outputs.date }}-${{ github.sha }} + restore-keys: | + scons-${{ steps.date.outputs.date }}- + scons- + + # build our docker image + - shell: bash + run: eval "$BUILD" diff --git a/.github/workflows/tools_tests.yaml b/.github/workflows/tools_tests.yaml index e93ce2bb39..173e208384 100644 --- a/.github/workflows/tools_tests.yaml +++ b/.github/workflows/tools_tests.yaml @@ -21,46 +21,6 @@ env: GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/comma_download_cache:/tmp/comma_download_cache $BASE_IMAGE /bin/sh -c jobs: - build_latest_ubuntu: - name: build latest ubuntu - runs-on: ubuntu-20.04 - timeout-minutes: 60 - steps: - - uses: actions/checkout@v3 - with: - submodules: true - - name: Cache pyenv - id: ubuntu-latest-pyenv - uses: actions/cache@v3 - with: - path: | - ~/.pyenv - ~/.local/share/virtualenvs/ - key: ubuntu-latest-python-${{ hashFiles('tools/ubuntu_setup.sh') }}- - - name: Cache scons - id: ubuntu-latest-scons - uses: actions/cache@v3 - with: - path: /tmp/scons_cache - key: ubuntu-latest-scons-${{ hashFiles('.github/workflows/tools_test.yaml') }}- - restore-keys: | - ubuntu-latest-scons-${{ hashFiles('.github/workflows/tools_test.yaml') }}- - ubuntu-latest-scons- - - - name: tools/ubuntu_setup.sh - run: | - source tools/openpilot_env.sh - tools/ubuntu_setup.sh - - name: Build openpilot - run: | - source tools/openpilot_env.sh - pipenv run scons -j$(nproc) --extras --test - - name: Cleanup scons cache - run: | - source tools/openpilot_env.sh - rm -rf /tmp/scons_cache/* - pipenv run scons -j$(nproc) --extras --test --cache-populate - plotjuggler: name: plotjuggler runs-on: ubuntu-20.04