From 917f71d4460bf5503190ade72a3fa760b64dca66 Mon Sep 17 00:00:00 2001
From: Justin Newberry <justin@comma.ai>
Date: Wed, 6 Sep 2023 11:11:15 -0700
Subject: [PATCH] CI: retry setup on failure (#29785)

* try a setup action

* should be uses

* fix that formatting

* try conclusion

* continue on error

* try without hyphens

* only when failure

* make it optional

* continue on error

* those don't fail anymore

* what about 3 failures

* remove stuff for debugging

* cleanup

* review suggestions

* change that too

* fix pj
---
 .github/workflows/badges.yaml                 |  2 +-
 .github/workflows/selfdrive_tests.yaml        | 18 ++++-----
 .../workflows/setup-with-retry/action.yaml    | 39 +++++++++++++++++++
 .github/workflows/setup/action.yaml           | 11 ++++++
 .github/workflows/tools_tests.yaml            |  6 +--
 5 files changed, 63 insertions(+), 13 deletions(-)
 create mode 100644 .github/workflows/setup-with-retry/action.yaml

diff --git a/.github/workflows/badges.yaml b/.github/workflows/badges.yaml
index 2ab3bded6a..a2c2daab1a 100644
--- a/.github/workflows/badges.yaml
+++ b/.github/workflows/badges.yaml
@@ -18,7 +18,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Push badges
       run: |
         ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/ui/translations/create_badges.py"
diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml
index b2c1bac822..a631069726 100644
--- a/.github/workflows/selfdrive_tests.yaml
+++ b/.github/workflows/selfdrive_tests.yaml
@@ -41,7 +41,7 @@ jobs:
     - name: Build devel
       timeout-minutes: 1
       run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Check submodules
       if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
       timeout-minutes: 1
@@ -74,7 +74,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Build openpilot with all flags
       timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 12 || 30) }} # allow more time when we missed the scons cache
       run: |
@@ -196,7 +196,7 @@ jobs:
       run: |
         echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
         $DOCKER_LOGIN
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
       with:
         git-lfs: false
     - name: Build and push CL Docker image
@@ -223,7 +223,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Build openpilot
       run: ${{ env.RUN }} "scons -j$(nproc)"
     - name: Run valgrind
@@ -241,7 +241,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Build openpilot
       timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
       run: ${{ env.RUN }} "scons -j$(nproc)"
@@ -273,7 +273,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Cache test routes
       id: dependency-cache
       uses: actions/cache@v3
@@ -313,7 +313,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Build base Docker image
       run: eval "$BUILD"
     - name: Build Docker image
@@ -349,7 +349,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Cache test routes
       id: dependency-cache
       uses: actions/cache@v3
@@ -378,7 +378,7 @@ jobs:
         with:
           submodules: true
           ref: ${{ github.event.pull_request.base.ref }}
-      - uses: ./.github/workflows/setup
+      - uses: ./.github/workflows/setup-with-retry
       - name: Get base car info
         run: |
           ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_info.py --path /tmp/openpilot_cache/base_car_info"
diff --git a/.github/workflows/setup-with-retry/action.yaml b/.github/workflows/setup-with-retry/action.yaml
new file mode 100644
index 0000000000..427b5a45ac
--- /dev/null
+++ b/.github/workflows/setup-with-retry/action.yaml
@@ -0,0 +1,39 @@
+name: 'openpilot env setup, with retry on failure'
+
+inputs:
+  git_lfs:
+    description: 'Whether or not to pull the git lfs'
+    required: false
+    default: 'true'
+
+env:
+  SLEEP_TIME: 30 # Time to sleep between retries
+
+runs:
+  using: "composite"
+  steps:
+    - id: setup1
+      uses: ./.github/workflows/setup
+      continue-on-error: true
+      with:
+        git_lfs: ${{ inputs.git_lfs }}
+        is_retried: true
+    - if: steps.setup1.outcome == 'failure'
+      shell: bash
+      run: sleep ${{ env.SLEEP_TIME }}
+    - id: setup2
+      if: steps.setup1.outcome == 'failure'
+      uses: ./.github/workflows/setup
+      continue-on-error: true
+      with:
+        git_lfs: ${{ inputs.git_lfs }}
+        is_retried: true
+    - if: steps.setup2.outcome == 'failure'
+      shell: bash
+      run: sleep ${{ env.SLEEP_TIME }}
+    - id: setup3
+      if: steps.setup2.outcome == 'failure'
+      uses: ./.github/workflows/setup
+      with:
+        git_lfs: ${{ inputs.git_lfs }}
+        is_retried: true
\ No newline at end of file
diff --git a/.github/workflows/setup/action.yaml b/.github/workflows/setup/action.yaml
index 263432ea98..74b82f9158 100644
--- a/.github/workflows/setup/action.yaml
+++ b/.github/workflows/setup/action.yaml
@@ -5,10 +5,21 @@ inputs:
     description: 'Whether or not to pull the git lfs'
     required: false
     default: 'true'
+  is_retried:
+    description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly'
+    required: false
+    default: 'false'
 
 runs:
   using: "composite"
   steps:
+    # assert that this action is retried using the setup-with-retry
+    - shell: bash
+      if: ${{ inputs.is_retried == 'false' }}
+      run: |
+        echo "You should not run this action directly. Use setup-with-retry instead"
+        exit 1
+
     # do this after checkout to ensure our custom LFS config is used to pull from GitLab
     - shell: bash
       if: ${{ inputs.git_lfs == 'true' }}
diff --git a/.github/workflows/tools_tests.yaml b/.github/workflows/tools_tests.yaml
index ccdbef0044..c7a5f80c3b 100644
--- a/.github/workflows/tools_tests.yaml
+++ b/.github/workflows/tools_tests.yaml
@@ -33,7 +33,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Build openpilot
       timeout-minutes: 5
       run: ${{ env.RUN }} "scons -j$(nproc) cereal/ common/ --minimal"
@@ -51,7 +51,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Build base cl image
       run: eval "$BUILD_CL"
     - name: Setup to push to repo
@@ -71,7 +71,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
       with:
         git_lfs: false
     - name: Setup to push to repo