From 7c378814f1dceca0498a8e4e827a1b7f87286736 Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Tue, 16 Apr 2024 12:00:18 -0700 Subject: [PATCH] jenkins: nightly casync build off of master (#32218) release node --- Jenkinsfile | 213 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 133 insertions(+), 80 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 909fbf63dd..f68ed86b66 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,6 +9,28 @@ def retryWithDelay(int maxRetries, int delay, Closure body) { throw Exception("Failed after ${maxRetries} retries") } +// check if started by timer: https://stackoverflow.com/questions/43516025/how-to-handle-nightly-build-in-jenkins-declarative-pipeline +@NonCPS +def isJobStartedByTimer() { + def startedByTimer = false + try { + def buildCauses = currentBuild.rawBuild.getCauses() + for ( buildCause in buildCauses ) { + if (buildCause != null) { + def causeDescription = buildCause.getShortDescription() + echo "shortDescription: ${causeDescription}" + if (causeDescription.contains("Started by timer")) { + startedByTimer = true + } + } + } + } catch(theError) { + echo "Error getting build cause" + } + + return startedByTimer +} + def device(String ip, String step_label, String cmd) { withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) { def ssh_cmd = """ @@ -151,15 +173,31 @@ def build_release(String channel_name) { deviceStage("build git", "tici-needs-can", [], [ ["build ${channel_name}", "RELEASE_BRANCH=${channel_name} $SOURCE_DIR/release/build_release.sh"], ]) + } + ) +} + + +def build_casync_release(String channel_name, def release) { + def extra_env = release ? "RELEASE=1" : "" + + return deviceStage("build casync", "tici-needs-can", [], [ + ["build", "${extra_env} BUILD_DIR=/data/openpilot CASYNC_DIR=/data/casync/openpilot $SOURCE_DIR/release/create_casync_build.sh"], + ["create manifest", "$SOURCE_DIR/release/create_release_manifest.py /data/openpilot /data/manifest.json && cat /data/manifest.json"], + ["upload and cleanup", "PYTHONWARNINGS=ignore $SOURCE_DIR/release/upload_casync_release.py /data/casync && rm -rf /data/casync"], + ]) +} + + +def build_stage() { + return parallel ( + 'nightly': { + build_release("nightly", true); }, - "${channel_name} (casync)": { - deviceStage("build casync", "tici-needs-can", [], [ - ["build", "RELEASE=1 BUILD_DIR=/data/openpilot CASYNC_DIR=/data/casync/openpilot $SOURCE_DIR/release/create_casync_build.sh"], - ["create manifest", "$SOURCE_DIR/release/create_release_manifest.py /data/openpilot /data/manifest.json && cat /data/manifest.json"], - ["upload and cleanup", "PYTHONWARNINGS=ignore $SOURCE_DIR/release/upload_casync_release.py /data/casync && rm -rf /data/casync"], - ]) + 'master': { + build_release("master", false); }, - "publish agnos": { + 'publish agnos': { pcStage("publish agnos") { sh "release/create_casync_agnos_release.py /tmp/casync/agnos /tmp/casync_tmp" sh "PYTHONWARNINGS=ignore ${env.WORKSPACE}/release/upload_casync_release.py /tmp/casync" @@ -168,7 +206,6 @@ def build_release(String channel_name) { ) } - node { env.CI = "1" env.PYTHONWARNINGS = "error" @@ -183,12 +220,23 @@ node { 'testing-closet*', 'hotfix-*'] def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*') - if (env.BRANCH_NAME != 'master') { - properties([ - disableConcurrentBuilds(abortPrevious: true) - ]) + def nightlyBranch = "master" + + def props = []; + + if (env.BRANCH_NAME == nightlyBranch) { + props.add(pipelineTriggers([ + pollSCM('* * * * *'), // every commit + cron('0 2 * * *') // and at 2am every night + ])) + } + + if (env.branch != "master") { + props.add(disableConcurrentBuilds(abortPrevious: true)) } + properties(props); + try { if (env.BRANCH_NAME == 'devel-staging') { build_release("release3-staging") @@ -199,74 +247,79 @@ node { } if (!env.BRANCH_NAME.matches(excludeRegex)) { - parallel ( - // tici tests - 'onroad tests': { - deviceStage("onroad", "tici-needs-can", [], [ - // TODO: ideally, this test runs in master-ci, but it takes 5+m to build it - //["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR $SOURCE_DIR/scripts/retry.sh ./build_devel.sh"], - ["build openpilot", "cd selfdrive/manager && ./build.py"], - ["check dirty", "release/check-dirty.sh"], - ["onroad tests", "pytest selfdrive/test/test_onroad.py -s"], - ["time to onroad", "pytest selfdrive/test/test_time_to_onroad.py"], - ]) - }, - 'HW + Unit Tests': { - deviceStage("tici-hardware", "tici-common", ["UNSAFE=1"], [ - ["build", "cd selfdrive/manager && ./build.py"], - ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"], - ["test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"], - ["test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py"], - ["test pigeond", "pytest system/ubloxd/tests/test_pigeond.py"], - ["test manager", "pytest selfdrive/manager/test/test_manager.py"], - ]) - }, - 'loopback': { - deviceStage("loopback", "tici-loopback", ["UNSAFE=1"], [ - ["build openpilot", "cd selfdrive/manager && ./build.py"], - ["test boardd loopback", "pytest selfdrive/boardd/tests/test_boardd_loopback.py"], - ]) - }, - 'camerad': { - deviceStage("AR0231", "tici-ar0231", ["UNSAFE=1"], [ - ["build", "cd selfdrive/manager && ./build.py"], - ["test camerad", "pytest system/camerad/test/test_camerad.py"], - ["test exposure", "pytest system/camerad/test/test_exposure.py"], - ]) - deviceStage("OX03C10", "tici-ox03c10", ["UNSAFE=1"], [ - ["build", "cd selfdrive/manager && ./build.py"], - ["test camerad", "pytest system/camerad/test/test_camerad.py"], - ["test exposure", "pytest system/camerad/test/test_exposure.py"], - ]) - }, - 'sensord': { - deviceStage("LSM + MMC", "tici-lsmc", ["UNSAFE=1"], [ - ["build", "cd selfdrive/manager && ./build.py"], - ["test sensord", "pytest system/sensord/tests/test_sensord.py"], - ]) - deviceStage("BMX + LSM", "tici-bmx-lsm", ["UNSAFE=1"], [ - ["build", "cd selfdrive/manager && ./build.py"], - ["test sensord", "pytest system/sensord/tests/test_sensord.py"], - ]) - }, - 'replay': { - deviceStage("model-replay", "tici-replay", ["UNSAFE=1"], [ - ["build", "cd selfdrive/manager && ./build.py"], - ["model replay", "selfdrive/test/process_replay/model_replay.py"], - ]) - }, - 'tizi': { - deviceStage("tizi", "tizi", ["UNSAFE=1"], [ - ["build openpilot", "cd selfdrive/manager && ./build.py"], - ["test boardd loopback", "SINGLE_PANDA=1 pytest selfdrive/boardd/tests/test_boardd_loopback.py"], - ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"], - ["test amp", "pytest system/hardware/tici/tests/test_amplifier.py"], - ["test hw", "pytest system/hardware/tici/tests/test_hardware.py"], - ["test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py"], - ]) - }, - - ) + parallel ( + // tici tests + 'onroad tests': { + deviceStage("onroad", "tici-needs-can", [], [ + // TODO: ideally, this test runs in master-ci, but it takes 5+m to build it + //["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR $SOURCE_DIR/scripts/retry.sh ./build_devel.sh"], + ["build openpilot", "cd selfdrive/manager && ./build.py"], + ["check dirty", "release/check-dirty.sh"], + ["onroad tests", "pytest selfdrive/test/test_onroad.py -s"], + ["time to onroad", "pytest selfdrive/test/test_time_to_onroad.py"], + ]) + }, + 'HW + Unit Tests': { + deviceStage("tici-hardware", "tici-common", ["UNSAFE=1"], [ + ["build", "cd selfdrive/manager && ./build.py"], + ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"], + ["test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"], + ["test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py"], + ["test pigeond", "pytest system/ubloxd/tests/test_pigeond.py"], + ["test manager", "pytest selfdrive/manager/test/test_manager.py"], + ]) + }, + 'loopback': { + deviceStage("loopback", "tici-loopback", ["UNSAFE=1"], [ + ["build openpilot", "cd selfdrive/manager && ./build.py"], + ["test boardd loopback", "pytest selfdrive/boardd/tests/test_boardd_loopback.py"], + ]) + }, + 'camerad': { + deviceStage("AR0231", "tici-ar0231", ["UNSAFE=1"], [ + ["build", "cd selfdrive/manager && ./build.py"], + ["test camerad", "pytest system/camerad/test/test_camerad.py"], + ["test exposure", "pytest system/camerad/test/test_exposure.py"], + ]) + deviceStage("OX03C10", "tici-ox03c10", ["UNSAFE=1"], [ + ["build", "cd selfdrive/manager && ./build.py"], + ["test camerad", "pytest system/camerad/test/test_camerad.py"], + ["test exposure", "pytest system/camerad/test/test_exposure.py"], + ]) + }, + 'sensord': { + deviceStage("LSM + MMC", "tici-lsmc", ["UNSAFE=1"], [ + ["build", "cd selfdrive/manager && ./build.py"], + ["test sensord", "pytest system/sensord/tests/test_sensord.py"], + ]) + deviceStage("BMX + LSM", "tici-bmx-lsm", ["UNSAFE=1"], [ + ["build", "cd selfdrive/manager && ./build.py"], + ["test sensord", "pytest system/sensord/tests/test_sensord.py"], + ]) + }, + 'replay': { + deviceStage("model-replay", "tici-replay", ["UNSAFE=1"], [ + ["build", "cd selfdrive/manager && ./build.py"], + ["model replay", "selfdrive/test/process_replay/model_replay.py"], + ]) + }, + 'tizi': { + deviceStage("tizi", "tizi", ["UNSAFE=1"], [ + ["build openpilot", "cd selfdrive/manager && ./build.py"], + ["test boardd loopback", "SINGLE_PANDA=1 pytest selfdrive/boardd/tests/test_boardd_loopback.py"], + ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"], + ["test amp", "pytest system/hardware/tici/tests/test_amplifier.py"], + ["test hw", "pytest system/hardware/tici/tests/test_hardware.py"], + ["test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py"], + ]) + }, + ) + } + + if (env.BRANCH_NAME == nightlyBranch && isJobStartedByTimer()) { + stage('build release') { + build_stage() + } } } catch (Exception e) { currentBuild.result = 'FAILED'