From 23cbfe9eb3fc3fd3655e41d44a63a20e04bae1f1 Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Thu, 18 Jan 2024 20:35:24 -0500 Subject: [PATCH] Revert "CI: use namespace.so" (#31060) --- .../workflows/auto-cache-restore/action.yaml | 34 ------------------- .../workflows/compile-openpilot/action.yaml | 8 ++++- .github/workflows/selfdrive_tests.yaml | 17 +++++----- .../workflows/setup-with-retry/action.yaml | 7 ++++ .github/workflows/setup/action.yaml | 25 +++++++++++--- .github/workflows/tools_tests.yaml | 2 +- 6 files changed, 44 insertions(+), 49 deletions(-) delete mode 100644 .github/workflows/auto-cache-restore/action.yaml diff --git a/.github/workflows/auto-cache-restore/action.yaml b/.github/workflows/auto-cache-restore/action.yaml deleted file mode 100644 index 0aab3b3eef..0000000000 --- a/.github/workflows/auto-cache-restore/action.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: 'automatically cache based on current runner' - -inputs: - path: - description: 'path to cache' - required: true - key: - description: 'key' - required: true - restore-keys: - description: 'restore-keys' - required: true - -runs: - using: "composite" - steps: - - name: setup namespace cache - if: ${{ contains(runner.name, 'nsc') }} - uses: namespacelabs/nscloud-cache-action@v1 - with: - path: ${{ inputs.path }} - - - name: setup github cache - if: ${{ !contains(runner.name, 'nsc') }} - uses: actions/cache/restore@v3 - with: - path: ${{ inputs.path }} - key: ${{ inputs.key }} - restore-keys: ${{ inputs.restore-keys }} - - # make the directory manually in case we didn't get a hit, so it doesn't fail on future steps - - id: scons-cache-setup - shell: bash - run: mkdir -p ${{ inputs.path }} diff --git a/.github/workflows/compile-openpilot/action.yaml b/.github/workflows/compile-openpilot/action.yaml index 2945b67d2e..8775c96262 100644 --- a/.github/workflows/compile-openpilot/action.yaml +++ b/.github/workflows/compile-openpilot/action.yaml @@ -1,5 +1,11 @@ name: 'compile openpilot' +inputs: + cache_key_prefix: + description: 'Prefix for caching key' + required: false + default: 'scons' + runs: using: "composite" steps: @@ -18,4 +24,4 @@ runs: if: github.ref == 'refs/heads/master' with: path: .ci_cache/scons_cache - key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }} + key: ${{ inputs.cache_key_prefix }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }} diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index dfd3570d4f..b9e80e9f74 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -75,7 +75,7 @@ jobs: ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && '["x86_64", "aarch64"]' || '["x86_64"]' ) }} - runs-on: ${{ (matrix.arch == 'aarch64') && 'namespace-profile-arm64-2x8' || 'ubuntu-20.04' }} + runs-on: ${{ (matrix.arch == 'aarch64') && 'buildjet-2vcpu-ubuntu-2204-arm' || 'ubuntu-20.04' }} steps: - uses: actions/checkout@v4 with: @@ -83,15 +83,18 @@ jobs: - uses: ./.github/workflows/setup-with-retry with: docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }} + cache_key_prefix: scons_${{ matrix.arch }} - uses: ./.github/workflows/compile-openpilot timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache + with: + cache_key_prefix: scons_${{ matrix.arch }} docker_push: name: docker push strategy: matrix: arch: ${{ fromJson( (github.repository == 'commaai/openpilot') && '["x86_64", "aarch64"]' || '["x86_64"]' ) }} - runs-on: ${{ (matrix.arch == 'aarch64') && 'namespace-profile-arm64-2x8' || 'ubuntu-20.04' }} + runs-on: ${{ (matrix.arch == 'aarch64') && 'buildjet-2vcpu-ubuntu-2204-arm' || 'ubuntu-20.04' }} if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' steps: - uses: actions/checkout@v4 @@ -130,9 +133,7 @@ jobs: static_analysis: name: static analysis - runs-on: ${{ ((github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-20.04' }} + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 with: @@ -164,7 +165,7 @@ jobs: name: unit tests runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-20.04' }} + (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'buildjet-8vcpu-ubuntu-2004' || 'ubuntu-20.04' }} steps: - uses: actions/checkout@v4 with: @@ -180,7 +181,7 @@ jobs: run: | ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \ export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && \ - $PYTEST --timeout 40 -m 'not slow' -n $(nproc) && \ + $PYTEST --timeout 40 -m 'not slow' && \ ./selfdrive/ui/tests/create_test_translations.sh && \ QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \ ./selfdrive/ui/tests/test_translations.py" @@ -195,7 +196,7 @@ jobs: name: process replay runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-20.04' }} + (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'buildjet-8vcpu-ubuntu-2004' || 'ubuntu-20.04' }} steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/setup-with-retry/action.yaml b/.github/workflows/setup-with-retry/action.yaml index 369ac45c9a..e0da746b44 100644 --- a/.github/workflows/setup-with-retry/action.yaml +++ b/.github/workflows/setup-with-retry/action.yaml @@ -5,6 +5,10 @@ inputs: description: 'Auth token for Docker Hub, required for BuildJet jobs' required: false default: '' + cache_key_prefix: + description: 'Prefix for caching key' + required: false + default: 'scons_x86_64' sleep_time: description: 'Time to sleep between retries' required: false @@ -18,6 +22,7 @@ runs: continue-on-error: true with: docker_hub_pat: ${{ inputs.docker_hub_pat }} + cache_key_prefix: ${{ inputs.cache_key_prefix }} is_retried: true - if: steps.setup1.outcome == 'failure' shell: bash @@ -28,6 +33,7 @@ runs: continue-on-error: true with: docker_hub_pat: ${{ inputs.docker_hub_pat }} + cache_key_prefix: ${{ inputs.cache_key_prefix }} is_retried: true - if: steps.setup2.outcome == 'failure' shell: bash @@ -37,4 +43,5 @@ runs: uses: ./.github/workflows/setup with: docker_hub_pat: ${{ inputs.docker_hub_pat }} + cache_key_prefix: ${{ inputs.cache_key_prefix }} is_retried: true diff --git a/.github/workflows/setup/action.yaml b/.github/workflows/setup/action.yaml index ec1626c01b..8bb1ccc376 100644 --- a/.github/workflows/setup/action.yaml +++ b/.github/workflows/setup/action.yaml @@ -5,6 +5,10 @@ inputs: description: 'Auth token for Docker Hub, required for BuildJet jobs' required: true default: '' + cache_key_prefix: + description: 'Prefix for caching key' + required: true + default: 'scons_x86_64' is_retried: description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly' required: false @@ -43,14 +47,19 @@ runs: run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV - shell: bash run: echo "$CACHE_COMMIT_DATE" - - id: scons-cache - uses: ./.github/workflows/auto-cache-restore + - id: restore-scons-cache + uses: actions/cache/restore@v3 with: path: .ci_cache/scons_cache - key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }} + key: ${{ inputs.cache_key_prefix }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }} restore-keys: | - scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }} - scons-${{ runner.arch }} + ${{ inputs.cache_key_prefix }}-${{ env.CACHE_COMMIT_DATE }}- + ${{ inputs.cache_key_prefix }}- + # if we didn't get a cache hit, make the directory manually so it doesn't fail on future steps + - id: scons-cache-setup + shell: bash + if: steps.restore-scons-cache.outputs.cache-hit != 'true' + run: mkdir -p $GITHUB_WORKSPACE/.ci_cache/scons_cache # as suggested here: https://github.com/moby/moby/issues/32816#issuecomment-910030001 - id: normalize-file-permissions shell: bash @@ -58,6 +67,12 @@ runs: run: | find . -type f -executable -not -perm 755 -exec chmod 755 {} \; find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \; + - id: setup-buildx-action + if: contains(runner.name, 'buildjet') + name: Set up Docker Buildx on buildjet to ensure a consistent cache + uses: docker/setup-buildx-action@v2 + with: + driver: docker-container # build our docker image - shell: bash run: eval ${{ env.BUILD }} \ No newline at end of file diff --git a/.github/workflows/tools_tests.yaml b/.github/workflows/tools_tests.yaml index 5cecfb7e4e..f549ece1b1 100644 --- a/.github/workflows/tools_tests.yaml +++ b/.github/workflows/tools_tests.yaml @@ -81,7 +81,7 @@ jobs: - name: Run dev container run: | mkdir -p /tmp/devcontainer_scons_cache/ - cp -r $GITHUB_WORKSPACE/.ci_cache/scons_cache/. /tmp/devcontainer_scons_cache/ + cp -r $GITHUB_WORKSPACE/.ci_cache/scons_cache/* /tmp/devcontainer_scons_cache/ devcontainer up --workspace-folder . - name: Test environment run: |