CI: retry setup on failure (#29785)

* try a setup action

* should be uses

* fix that formatting

* try conclusion

* continue on error

* try without hyphens

* only when failure

* make it optional

* continue on error

* those don't fail anymore

* what about 3 failures

* remove stuff for debugging

* cleanup

* review suggestions

* change that too

* fix pj

.github/workflows/badges.yaml

@@ -18,7 +18,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Push badges
       run: |
         ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/ui/translations/create_badges.py"

.github/workflows/selfdrive_tests.yaml

@@ -41,7 +41,7 @@ jobs:
     - name: Build devel
       timeout-minutes: 1
       run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Check submodules
       if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
       timeout-minutes: 1
@@ -74,7 +74,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Build openpilot with all flags
       timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 12 || 30) }} # allow more time when we missed the scons cache
       run: |
@@ -196,7 +196,7 @@ jobs:
       run: |
         echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
         $DOCKER_LOGIN
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
       with:
         git-lfs: false
     - name: Build and push CL Docker image
@@ -223,7 +223,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Build openpilot
       run: ${{ env.RUN }} "scons -j$(nproc)"
     - name: Run valgrind
@@ -241,7 +241,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Build openpilot
       timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
       run: ${{ env.RUN }} "scons -j$(nproc)"
@@ -273,7 +273,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Cache test routes
       id: dependency-cache
       uses: actions/cache@v3
@@ -313,7 +313,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Build base Docker image
       run: eval "$BUILD"
     - name: Build Docker image
@@ -349,7 +349,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Cache test routes
       id: dependency-cache
       uses: actions/cache@v3
@@ -378,7 +378,7 @@ jobs:
         with:
           submodules: true
           ref: ${{ github.event.pull_request.base.ref }}
-      - uses: ./.github/workflows/setup
+      - uses: ./.github/workflows/setup-with-retry
       - name: Get base car info
         run: |
           ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_info.py --path /tmp/openpilot_cache/base_car_info"

.github/workflows/setup-with-retry/action.yaml

@@ -0,0 +1,39 @@
+name: 'openpilot env setup, with retry on failure'
+
+inputs:
+  git_lfs:
+    description: 'Whether or not to pull the git lfs'
+    required: false
+    default: 'true'
+
+env:
+  SLEEP_TIME: 30 # Time to sleep between retries
+
+runs:
+  using: "composite"
+  steps:
+    - id: setup1
+      uses: ./.github/workflows/setup
+      continue-on-error: true
+      with:
+        git_lfs: ${{ inputs.git_lfs }}
+        is_retried: true
+    - if: steps.setup1.outcome == 'failure'
+      shell: bash
+      run: sleep ${{ env.SLEEP_TIME }}
+    - id: setup2
+      if: steps.setup1.outcome == 'failure'
+      uses: ./.github/workflows/setup
+      continue-on-error: true
+      with:
+        git_lfs: ${{ inputs.git_lfs }}
+        is_retried: true
+    - if: steps.setup2.outcome == 'failure'
+      shell: bash
+      run: sleep ${{ env.SLEEP_TIME }}
+    - id: setup3
+      if: steps.setup2.outcome == 'failure'
+      uses: ./.github/workflows/setup
+      with:
+        git_lfs: ${{ inputs.git_lfs }}
+        is_retried: true

.github/workflows/setup/action.yaml

@@ -5,10 +5,21 @@ inputs:
     description: 'Whether or not to pull the git lfs'
     required: false
     default: 'true'
+  is_retried:
+    description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly'
+    required: false
+    default: 'false'
 
 runs:
   using: "composite"
   steps:
+    # assert that this action is retried using the setup-with-retry
+    - shell: bash
+      if: ${{ inputs.is_retried == 'false' }}
+      run: |
+        echo "You should not run this action directly. Use setup-with-retry instead"
+        exit 1
+
     # do this after checkout to ensure our custom LFS config is used to pull from GitLab
     - shell: bash
       if: ${{ inputs.git_lfs == 'true' }}

.github/workflows/tools_tests.yaml

@@ -33,7 +33,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Build openpilot
       timeout-minutes: 5
       run: ${{ env.RUN }} "scons -j$(nproc) cereal/ common/ --minimal"
@@ -51,7 +51,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Build base cl image
       run: eval "$BUILD_CL"
     - name: Setup to push to repo
@@ -71,7 +71,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
       with:
         git_lfs: false
     - name: Setup to push to repo