From 12da944f1c4bc4909992a73419e79f8afbefb477 Mon Sep 17 00:00:00 2001
From: Justin Newberry <justin@comma.ai>
Date: Wed, 6 Dec 2023 17:22:32 -0800
Subject: [PATCH] jenkins: retry build_devel (#30628)

* will it work

* retry build devel

* correct p[ath

* try a failure

* Revert "try a failure"

This reverts commit ba4e6a0a2096b94887fcfbf7d011e34aa2a16c41.
old-commit-hash: 36bebb1aa064b4b40580aa48d35b21bd9ebe14c2
---
 Jenkinsfile            |  2 +-
 release/build_devel.sh |  1 +
 scripts/retry.sh       | 27 +++++++++++++++++++++++++++
 3 files changed, 29 insertions(+), 1 deletion(-)
 create mode 100755 scripts/retry.sh

diff --git a/Jenkinsfile b/Jenkinsfile
index ec39882201..840dd29503 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -158,7 +158,7 @@ node {
       // tici tests
       'onroad tests': {
         deviceStage("onroad", "tici-needs-can", ["SKIP_COPY=1"], [
-          ["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR ./build_devel.sh"],
+          ["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR $SOURCE_DIR/scripts/retry.sh ./build_devel.sh"],
           ["build openpilot", "cd selfdrive/manager && ./build.py"],
           ["check dirty", "release/check-dirty.sh"],
           ["onroad tests", "pytest selfdrive/test/test_onroad.py -s"],
diff --git a/release/build_devel.sh b/release/build_devel.sh
index ca04c56f1e..8b6816e423 100755
--- a/release/build_devel.sh
+++ b/release/build_devel.sh
@@ -22,6 +22,7 @@ pre-commit uninstall || true
 
 echo "[-] bringing master-ci and devel in sync T=$SECONDS"
 cd $TARGET_DIR
+
 git fetch --depth 1 origin master-ci
 git fetch --depth 1 origin devel
 
diff --git a/scripts/retry.sh b/scripts/retry.sh
new file mode 100755
index 0000000000..4861748e55
--- /dev/null
+++ b/scripts/retry.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+function fail {
+  echo $1 >&2
+  exit 1
+}
+
+function retry {
+  local n=1
+  local max=3 # 3 retries before failure
+  local delay=5 # delay between retries, 5 seconds
+  while true; do
+    echo "Running command '$@' with retry, attempt $n/$max"
+    "$@" && break || {
+      if [[ $n -lt $max ]]; then
+        ((n++))
+        sleep $delay;
+      else
+        fail "The command has failed after $n attempts."
+      fi
+    }
+  done
+}
+
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+    retry "$@"
+fi