update release build (#21482)

* update master-ci

* move to /data

* fix tests

* fix source dir

* prune

* trailing slash

* compile
old-commit-hash: 3a51a5c0f0

Jenkinsfile

@@ -7,6 +7,7 @@ set -e
 
 export CI=1
 export TEST_DIR=${env.TEST_DIR}
+export SOURCE_DIR=${env.SOURCE_DIR}
 export GIT_BRANCH=${env.GIT_BRANCH}
 export GIT_COMMIT=${env.GIT_COMMIT}
 
@@ -50,6 +51,7 @@ pipeline {
   environment {
     COMMA_JWT = credentials('athena-test-jwt')
     TEST_DIR = "/data/openpilot"
+    SOURCE_DIR = "/data/openpilot_source/"
   }
   options {
       timeout(time: 3, unit: 'HOURS')
@@ -127,7 +129,8 @@ pipeline {
                 stage('Devel Tests') {
                   steps {
                     phone_steps("eon-build", [
-                      ["build devel", "cd release && DEVEL_TEST=1 ./build_devel.sh"],
+                      ["build devel", "cd $SOURCE_DIR/release && EXTRA_FILES='tools/' ./build_devel.sh"],
+                      ["build openpilot", "cd selfdrive/manager && ./build.py"],
                       ["test manager", "python selfdrive/manager/test/test_manager.py"],
                       ["onroad tests", "cd selfdrive/test/ && ./test_onroad.py"],
                       ["test car interfaces", "cd selfdrive/car/tests/ && ./test_car_interfaces.py"],
@@ -232,7 +235,7 @@ pipeline {
               }
               steps {
                 phone_steps("eon-build", [
-                  ["push devel", "cd release && CI_PUSH='master-ci' ./build_devel.sh"],
+                  ["push devel", "cd $SOURCE_DIR/release && PUSH='master-ci' ./build_devel.sh"],
                 ])
               }
             }

release/build_devel.sh

@@ -1,15 +1,12 @@
 #!/usr/bin/bash -e
 
-SOURCE_DIR=/data/openpilot_source
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
-TARGET_DIR=/data/openpilot
 
-ln -sf $TARGET_DIR /data/pythonpath
+TARGET_DIR=/data/openpilot
+SOURCE_DIR="$(git rev-parse --show-toplevel)"
 
-export GIT_COMMITTER_NAME="Vehicle Researcher"
+# set git identity
-export GIT_COMMITTER_EMAIL="user@comma.ai"
+source $DIR/identity.sh
-export GIT_AUTHOR_NAME="Vehicle Researcher"
-export GIT_AUTHOR_EMAIL="user@comma.ai"
-export GIT_SSH_COMMAND="ssh -i /data/gitkey"
 
 echo "[-] Setting up repo T=$SECONDS"
 if [ ! -d "$TARGET_DIR" ]; then
@@ -19,12 +16,10 @@ if [ ! -d "$TARGET_DIR" ]; then
   git remote add origin git@github.com:commaai/openpilot.git
 fi
 
-echo "[-] fetching public T=$SECONDS"
+echo "[-] bringing master-ci and devel in sync T=$SECONDS"
 cd $TARGET_DIR
 git prune || true
 git remote prune origin || true
-
-echo "[-] bringing master-ci and devel in sync T=$SECONDS"
 git fetch origin master-ci
 git fetch origin devel
 
@@ -38,27 +33,27 @@ git clean -xdf
 echo "[-] erasing old openpilot T=$SECONDS"
 find . -maxdepth 1 -not -path './.git' -not -name '.' -not -name '..' -exec rm -rf '{}' \;
 
-# reset tree and get version
+# reset source tree
 cd $SOURCE_DIR
 git clean -xdf
-git checkout -- selfdrive/common/version.h
-
-VERSION=$(cat selfdrive/common/version.h | awk -F\" '{print $2}')
-echo "#define COMMA_VERSION \"$VERSION-$(git --git-dir=$SOURCE_DIR/.git rev-parse --short HEAD)-$(date '+%Y-%m-%dT%H:%M:%S')\"" > selfdrive/common/version.h
 
 # do the files copy
 echo "[-] copying files T=$SECONDS"
 cd $SOURCE_DIR
 cp -pR --parents $(cat release/files_common) $TARGET_DIR/
-
+#cp -pR --parents $(cat release/files_tici) $TARGET_DIR/
-# test files
+if [ ! -z "$EXTRA_FILES" ]; then
-if [ ! -z "$DEVEL_TEST" ]; then
+  cp -pR --parents $EXTRA_FILES $TARGET_DIR/
-  cp -pR --parents tools/ $TARGET_DIR/
 fi
 
+# append source commit hash and build date to version
+GIT_HASH=$(git --git-dir=$SOURCE_DIR/.git rev-parse --short HEAD)
+DATETIME=$(date '+%Y-%m-%dT%H:%M:%S')
+VERSION=$(cat selfdrive/common/version.h | awk -F\" '{print $2}')
+echo "#define COMMA_VERSION \"$VERSION-$GIT_HASH-$DATETIME\"" > selfdrive/common/version.h
+
 # in the directory
 cd $TARGET_DIR
-
 rm -f panda/board/obj/panda.bin.signed
 
 echo "[-] committing version $VERSION T=$SECONDS"
@@ -66,13 +61,10 @@ git add -f .
 git status
 git commit -a -m "openpilot v$VERSION release"
 
-# Run build
+if [ ! -z "$PUSH" ]; then
-selfdrive/manager/build.py
+  echo "[-] Pushing to $PUSH T=$SECONDS"
-
-if [ ! -z "$CI_PUSH" ]; then
-  echo "[-] Pushing to $CI_PUSH T=$SECONDS"
   git remote set-url origin git@github.com:commaai/openpilot.git
-  git push -f origin master-ci:$CI_PUSH
+  git push -f origin master-ci:$PUSH
 fi
 
 echo "[-] done T=$SECONDS"

release/build_release3.sh

@@ -1,16 +1,15 @@
 #!/usr/bin/bash -e
 
-export GIT_COMMITTER_NAME="Vehicle Researcher"
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
-export GIT_COMMITTER_EMAIL="user@comma.ai"
-export GIT_AUTHOR_NAME="Vehicle Researcher"
-export GIT_AUTHOR_EMAIL="user@comma.ai"
-export GIT_SSH_COMMAND="ssh -i /data/gitkey"
 
 BUILD_DIR=/data/releasepilot
 SOURCE_DIR="$(git rev-parse --show-toplevel)"
 
 BRANCH=release3-staging
 
+# set git identity
+source $DIR/identity.sh
+
 echo "[-] Setting up repo T=$SECONDS"
 rm -rf $BUILD_DIR
 mkdir -p $BUILD_DIR

release/identity.sh

@@ -0,0 +1,5 @@
+export GIT_COMMITTER_NAME="Vehicle Researcher"
+export GIT_COMMITTER_EMAIL="user@comma.ai"
+export GIT_AUTHOR_NAME="Vehicle Researcher"
+export GIT_AUTHOR_EMAIL="user@comma.ai"
+export GIT_SSH_COMMAND="ssh -i /data/gitkey"

selfdrive/test/setup_device_ci.sh

@@ -1,6 +1,9 @@
 #!/usr/bin/bash -e
 
-export SOURCE_DIR="/data/openpilot_source/"
+if [ -z "$SOURCE_DIR" ]; then
+  echo "SOURCE_DIR must be set"
+  exit 1
+fi
 
 if [ -z "$GIT_COMMIT" ]; then
   echo "GIT_COMMIT must be set"