diff --git a/Jenkinsfile b/Jenkinsfile index ec39882201..840dd29503 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -158,7 +158,7 @@ node { // tici tests 'onroad tests': { deviceStage("onroad", "tici-needs-can", ["SKIP_COPY=1"], [ - ["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR ./build_devel.sh"], + ["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR $SOURCE_DIR/scripts/retry.sh ./build_devel.sh"], ["build openpilot", "cd selfdrive/manager && ./build.py"], ["check dirty", "release/check-dirty.sh"], ["onroad tests", "pytest selfdrive/test/test_onroad.py -s"], diff --git a/release/build_devel.sh b/release/build_devel.sh index ca04c56f1e..8b6816e423 100755 --- a/release/build_devel.sh +++ b/release/build_devel.sh @@ -22,6 +22,7 @@ pre-commit uninstall || true echo "[-] bringing master-ci and devel in sync T=$SECONDS" cd $TARGET_DIR + git fetch --depth 1 origin master-ci git fetch --depth 1 origin devel diff --git a/scripts/retry.sh b/scripts/retry.sh new file mode 100755 index 0000000000..4861748e55 --- /dev/null +++ b/scripts/retry.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +function fail { + echo $1 >&2 + exit 1 +} + +function retry { + local n=1 + local max=3 # 3 retries before failure + local delay=5 # delay between retries, 5 seconds + while true; do + echo "Running command '$@' with retry, attempt $n/$max" + "$@" && break || { + if [[ $n -lt $max ]]; then + ((n++)) + sleep $delay; + else + fail "The command has failed after $n attempts." + fi + } + done +} + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + retry "$@" +fi