| 
						
						
							
								
							
						
						
					 | 
					 | 
					@ -9,6 +9,28 @@ def retryWithDelay(int maxRetries, int delay, Closure body) { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					  throw Exception("Failed after ${maxRetries} retries") | 
					 | 
					 | 
					 | 
					  throw Exception("Failed after ${maxRetries} retries") | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					} | 
					 | 
					 | 
					 | 
					} | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					
 | 
					 | 
					 | 
					 | 
					
 | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					// check if started by timer: https://stackoverflow.com/questions/43516025/how-to-handle-nightly-build-in-jenkins-declarative-pipeline | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					@NonCPS | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					def isJobStartedByTimer() { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					  def startedByTimer = false | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					  try { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					    def buildCauses = currentBuild.rawBuild.getCauses() | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					    for ( buildCause in buildCauses ) { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					      if (buildCause != null) { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					        def causeDescription = buildCause.getShortDescription() | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					        echo "shortDescription: ${causeDescription}" | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					        if (causeDescription.contains("Started by timer")) { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					          startedByTimer = true | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					        } | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					      } | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					    } | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					  } catch(theError) { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					    echo "Error getting build cause" | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					  } | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					
 | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					  return startedByTimer | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					} | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					
 | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					def device(String ip, String step_label, String cmd) { | 
					 | 
					 | 
					 | 
					def device(String ip, String step_label, String cmd) { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					  withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) { | 
					 | 
					 | 
					 | 
					  withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					    def ssh_cmd = """ | 
					 | 
					 | 
					 | 
					    def ssh_cmd = """ | 
				
			
			
		
	
	
		
		
			
				
					| 
						
							
								
							
						
						
							
								
							
						
						
					 | 
					 | 
					@ -151,15 +173,31 @@ def build_release(String channel_name) { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					      deviceStage("build git", "tici-needs-can", [], [ | 
					 | 
					 | 
					 | 
					      deviceStage("build git", "tici-needs-can", [], [ | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					        ["build ${channel_name}", "RELEASE_BRANCH=${channel_name} $SOURCE_DIR/release/build_release.sh"], | 
					 | 
					 | 
					 | 
					        ["build ${channel_name}", "RELEASE_BRANCH=${channel_name} $SOURCE_DIR/release/build_release.sh"], | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					      ]) | 
					 | 
					 | 
					 | 
					      ]) | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					    }, | 
					 | 
					 | 
					 | 
					    } | 
				
			
			
				
				
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					    "${channel_name} (casync)": { | 
					 | 
					 | 
					 | 
					  ) | 
				
			
			
				
				
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					      deviceStage("build casync", "tici-needs-can", [], [ | 
					 | 
					 | 
					 | 
					} | 
				
			
			
				
				
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					        ["build", "RELEASE=1 BUILD_DIR=/data/openpilot CASYNC_DIR=/data/casync/openpilot $SOURCE_DIR/release/create_casync_build.sh"], | 
					 | 
					 | 
					 | 
					
 | 
				
			
			
				
				
			
		
	
		
		
	
		
		
	
		
		
	
		
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					
 | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					def build_casync_release(String channel_name, def release) { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					  def extra_env = release ? "RELEASE=1" : "" | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					
 | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					  return deviceStage("build casync", "tici-needs-can", [], [ | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					    ["build", "${extra_env} BUILD_DIR=/data/openpilot CASYNC_DIR=/data/casync/openpilot $SOURCE_DIR/release/create_casync_build.sh"], | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					    ["create manifest", "$SOURCE_DIR/release/create_release_manifest.py /data/openpilot /data/manifest.json && cat /data/manifest.json"], | 
					 | 
					 | 
					 | 
					    ["create manifest", "$SOURCE_DIR/release/create_release_manifest.py /data/openpilot /data/manifest.json && cat /data/manifest.json"], | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					    ["upload and cleanup", "PYTHONWARNINGS=ignore $SOURCE_DIR/release/upload_casync_release.py /data/casync && rm -rf /data/casync"], | 
					 | 
					 | 
					 | 
					    ["upload and cleanup", "PYTHONWARNINGS=ignore $SOURCE_DIR/release/upload_casync_release.py /data/casync && rm -rf /data/casync"], | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					  ]) | 
					 | 
					 | 
					 | 
					  ]) | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					} | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					
 | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					
 | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					def build_stage() { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					  return parallel ( | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					    'nightly': { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					      build_release("nightly", true); | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					    }, | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					    'master': { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					      build_release("master", false); | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					    }, | 
					 | 
					 | 
					 | 
					    }, | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					    "publish agnos": { | 
					 | 
					 | 
					 | 
					    'publish agnos': { | 
				
			
			
				
				
			
		
	
		
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					      pcStage("publish agnos") { | 
					 | 
					 | 
					 | 
					      pcStage("publish agnos") { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					        sh "release/create_casync_agnos_release.py /tmp/casync/agnos /tmp/casync_tmp" | 
					 | 
					 | 
					 | 
					        sh "release/create_casync_agnos_release.py /tmp/casync/agnos /tmp/casync_tmp" | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					        sh "PYTHONWARNINGS=ignore ${env.WORKSPACE}/release/upload_casync_release.py /tmp/casync" | 
					 | 
					 | 
					 | 
					        sh "PYTHONWARNINGS=ignore ${env.WORKSPACE}/release/upload_casync_release.py /tmp/casync" | 
				
			
			
		
	
	
		
		
			
				
					| 
						
						
						
							
								
							
						
					 | 
					 | 
					@ -168,7 +206,6 @@ def build_release(String channel_name) { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					  ) | 
					 | 
					 | 
					 | 
					  ) | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					} | 
					 | 
					 | 
					 | 
					} | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					
 | 
					 | 
					 | 
					 | 
					
 | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					
 | 
					 | 
					 | 
					 | 
					 | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					node { | 
					 | 
					 | 
					 | 
					node { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					  env.CI = "1" | 
					 | 
					 | 
					 | 
					  env.CI = "1" | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					  env.PYTHONWARNINGS = "error" | 
					 | 
					 | 
					 | 
					  env.PYTHONWARNINGS = "error" | 
				
			
			
		
	
	
		
		
			
				
					| 
						
						
						
							
								
							
						
					 | 
					 | 
					@ -183,12 +220,23 @@ node { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					                         'testing-closet*', 'hotfix-*'] | 
					 | 
					 | 
					 | 
					                         'testing-closet*', 'hotfix-*'] | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					  def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*') | 
					 | 
					 | 
					 | 
					  def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*') | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					
 | 
					 | 
					 | 
					 | 
					
 | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					  if (env.BRANCH_NAME != 'master') { | 
					 | 
					 | 
					 | 
					  def nightlyBranch = "master" | 
				
			
			
				
				
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					    properties([ | 
					 | 
					 | 
					 | 
					
 | 
				
			
			
				
				
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					        disableConcurrentBuilds(abortPrevious: true) | 
					 | 
					 | 
					 | 
					  def props = []; | 
				
			
			
				
				
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					    ]) | 
					 | 
					 | 
					 | 
					
 | 
				
			
			
				
				
			
		
	
		
		
	
		
		
	
		
		
	
		
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					  if (env.BRANCH_NAME == nightlyBranch) { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					    props.add(pipelineTriggers([ | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					      pollSCM('* * * * *'), // every commit | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					      cron('0 2 * * *')     // and at 2am every night | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					    ])) | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					  } | 
					 | 
					 | 
					 | 
					  } | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					
 | 
					 | 
					 | 
					 | 
					
 | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					  if (env.branch != "master") { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					    props.add(disableConcurrentBuilds(abortPrevious: true)) | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					  } | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					
 | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					  properties(props); | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					
 | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					  try { | 
					 | 
					 | 
					 | 
					  try { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					    if (env.BRANCH_NAME == 'devel-staging') { | 
					 | 
					 | 
					 | 
					    if (env.BRANCH_NAME == 'devel-staging') { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					      build_release("release3-staging") | 
					 | 
					 | 
					 | 
					      build_release("release3-staging") | 
				
			
			
		
	
	
		
		
			
				
					| 
						
							
								
							
						
						
							
								
							
						
						
					 | 
					 | 
					@ -265,9 +313,14 @@ node { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					            ["test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py"], | 
					 | 
					 | 
					 | 
					            ["test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py"], | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					          ]) | 
					 | 
					 | 
					 | 
					          ]) | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					        }, | 
					 | 
					 | 
					 | 
					        }, | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					
 | 
					 | 
					 | 
					 | 
					 | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					      ) | 
					 | 
					 | 
					 | 
					      ) | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					    } | 
					 | 
					 | 
					 | 
					    } | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					
 | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					    if (env.BRANCH_NAME == nightlyBranch && isJobStartedByTimer()) { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					      stage('build release') { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					        build_stage() | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					      } | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					 | 
					    } | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					  } catch (Exception e) { | 
					 | 
					 | 
					 | 
					  } catch (Exception e) { | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					    currentBuild.result = 'FAILED' | 
					 | 
					 | 
					 | 
					    currentBuild.result = 'FAILED' | 
				
			
			
		
	
		
		
			
				
					
					 | 
					 | 
					 | 
					    throw e | 
					 | 
					 | 
					 | 
					    throw e | 
				
			
			
		
	
	
		
		
			
				
					| 
						
							
								
							
						
						
						
					 | 
					 | 
					
  |