From 93cd19e3a553d34d2f8bcd1aea121d09a7b2196b Mon Sep 17 00:00:00 2001
From: Justin Newberry <justin@comma.ai>
Date: Tue, 12 Dec 2023 17:54:17 -0800
Subject: [PATCH] jenkins: retry pc setup (#30659)

* retry pc setup

* sleep and retry

* exception

* throw e

* remove force exit

* make a function for it

* remove force exit

* move to top

* separate exception
---
 Jenkinsfile | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 4e8575a0d8..84f87bbdd9 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,3 +1,15 @@
+def retryWithDelay(int maxRetries, int delay, Closure body) {
+  for (int i = 0; i < maxRetries; i++) {
+    try {
+      body()
+      return
+    } catch (Exception e) {
+      sleep(delay)
+    }
+  }
+  throw Exception("Failed after ${maxRetries} retries")
+}
+
 def device(String ip, String step_label, String cmd) {
   withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) {
     def ssh_cmd = """
@@ -92,18 +104,20 @@ def pcStage(String stageName, Closure body) {
     docker.build("openpilot-base:build-${env.GIT_COMMIT}", "-f Dockerfile.openpilot_base .").inside(dockerArgs) {
       timeout(time: 20, unit: 'MINUTES') {
         try {
-          sh "git config --global --add safe.directory '*'"
-          sh "git submodule update --init --recursive"
-          sh "git lfs pull"
+          retryWithDelay (3, 15) {
+            sh "git config --global --add safe.directory '*'"
+            sh "git submodule update --init --recursive"
+            sh "git lfs pull"
+          }
           body()
         } finally {
-          sh "rm -rf ${env.WORKSPACE}/* || true"
-          sh "rm -rf .* || true"
+            sh "rm -rf ${env.WORKSPACE}/* || true"
+            sh "rm -rf .* || true"
+          }
         }
       }
     }
   }
-  }
 }
 
 def setupCredentials() {