run onroad tests on release build (#22700)

* check startup aelrt

* release decorator

* run in jenkins

* run onroad tests

* no push

* fix release2 build

* fix path

* no overwrite

* single release build script

* make files_eon non empty

* files

* run in source

* add that for now

* print

* ignore initialzing

* print

* fetch

* run tests last

* check alert text

* revert some stuff

* fixup jenkinsfile

Co-authored-by: Comma Device <device@comma.ai>

Jenkinsfile

@@ -70,7 +70,7 @@ pipeline {
       }
       steps {
         phone_steps("eon-build", [
-          ["build release2-staging & dashcam-staging", "cd release && PUSH=1 ./build_release2.sh"],
+          ["build release2-staging & dashcam-staging", "cd release && PUSH=1 ./build_release.sh"],
         ])
       }
     }
@@ -87,7 +87,7 @@ pipeline {
       }
       steps {
         phone_steps("tici", [
-          ["build release3-staging & dashcam3-staging", "PUSH=1 $SOURCE_DIR/release/build_release3.sh"],
+          ["build release3-staging & dashcam3-staging", "PUSH=1 $SOURCE_DIR/release/build_release.sh"],
         ])
       }
     }

release/build_release.sh

@@ -9,7 +9,17 @@ cd $DIR
 BUILD_DIR=/data/openpilot
 SOURCE_DIR="$(git rev-parse --show-toplevel)"
 
-BRANCH=release3-staging
+if [ -f /TICI ]; then
+  FILES_SRC="release/files_tici"
+  RELEASE_BRANCH=release3-staging
+  DASHBAM_BRANCH=dashcam3-staging
+elif [ -f /EON ]; then
+  FILES_SRC="release/files_eon"
+  RELEASE_BRANCH=release2-staging
+  DASHBAM_BRANCH=dashcam-staging
+else
+  exit 0
+fi
 
 # set git identity
 source $DIR/identity.sh
@@ -20,13 +30,14 @@ mkdir -p $BUILD_DIR
 cd $BUILD_DIR
 git init
 git remote add origin git@github.com:commaai/openpilot.git
-git checkout -f -B $BRANCH
+git fetch origin $RELEASE_BRANCH
+git checkout --orphan $RELEASE_BRANCH
 
 # do the files copy
 echo "[-] copying files T=$SECONDS"
 cd $SOURCE_DIR
 cp -pR --parents $(cat release/files_common) $BUILD_DIR/
-cp -pR --parents $(cat release/files_tici) $BUILD_DIR/
+cp -pR --parents $(cat $FILES_SRC) $BUILD_DIR/
 
 # in the directory
 cd $BUILD_DIR
@@ -39,6 +50,7 @@ echo "#define COMMA_VERSION \"$VERSION-release\"" > selfdrive/common/version.h
 echo "[-] committing version $VERSION T=$SECONDS"
 git add -f .
 git commit -a -m "openpilot v$VERSION release"
+git branch --set-upstream-to=origin/$RELEASE_BRANCH
 
 # Build panda firmware
 pushd panda/
@@ -50,15 +62,20 @@ popd
 export PYTHONPATH="$BUILD_DIR"
 scons -j$(nproc)
 
-# Run tests
+# Ensure no submodules in release
-#python selfdrive/manager/test/test_manager.py
+if test "$(git submodule--helper list | wc -l)" -gt "0"; then
-selfdrive/car/tests/test_car_interfaces.py
+  echo "submodules found:"
+  git submodule--helper list
+  exit 1
+fi
+git submodule status
 
 # Cleanup
 find . -name '*.a' -delete
 find . -name '*.o' -delete
 find . -name '*.os' -delete
 find . -name '*.pyc' -delete
+find . -name 'moc_*' -delete
 find . -name '__pycache__' -delete
 rm -rf panda/board panda/certs panda/crypto
 rm -rf .sconsign.dblite Jenkinsfile release/
@@ -78,15 +95,24 @@ touch prebuilt
 git add -f .
 git commit --amend -m "openpilot v$VERSION"
 
+# Run tests
+TEST_FILES="tools/"
+cd $SOURCE_DIR
+cp -pR -n --parents $TEST_FILES $BUILD_DIR/
+cd $BUILD_DIR
+RELEASE=1 selfdrive/test/test_onroad.py
+#selfdrive/manager/test/test_manager.py
+selfdrive/car/tests/test_car_interfaces.py
+rm -rf $TEST_FILES
+
 if [ ! -z "$PUSH" ]; then
   echo "[-] pushing T=$SECONDS"
-  git remote set-url origin git@github.com:commaai/openpilot.git
+  git push -f origin $RELEASE_BRANCH
-  git push -f origin $BRANCH
 
   # Create dashcam
   git rm selfdrive/car/*/carcontroller.py
   git commit -m "create dashcam release from release"
-  git push -f origin $BRANCH:dashcam3-staging
+  git push -f origin $RELEASE_BRANCH:$DASHCAM_BRANCH
 fi
 
 echo "[-] done T=$SECONDS"

release/build_release2.sh

@@ -1,104 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-export GIT_COMMITTER_NAME="Vehicle Researcher"
-export GIT_COMMITTER_EMAIL="user@comma.ai"
-export GIT_AUTHOR_NAME="Vehicle Researcher"
-export GIT_AUTHOR_EMAIL="user@comma.ai"
-
-export GIT_SSH_COMMAND="ssh -i /data/gitkey"
-
-# set CLEAN to build outside of CI
-if [ ! -z "$CLEAN" ]; then
-  # Create folders
-  rm -rf /data/openpilot
-  mkdir -p /data/openpilot
-  cd /data/openpilot
-
-  # Create git repo
-  git init
-  git remote add origin git@github.com:commaai/openpilot.git
-  git fetch origin devel-staging
-else
-  cd /data/openpilot
-  git clean -xdf
-  git branch -D release2-staging || true
-fi
-
-git fetch origin release2-staging
-git fetch origin dashcam-staging
-
-# Create release2 with no history
-if [ ! -z "$CLEAN" ]; then
-  git checkout --orphan release2-staging origin/devel-staging
-else
-  git checkout --orphan release2-staging
-fi
-
-VERSION=$(cat selfdrive/common/version.h | awk -F[\"-]  '{print $2}')
-echo "#define COMMA_VERSION \"$VERSION-release\"" > selfdrive/common/version.h
-
-git commit -m "openpilot v$VERSION"
-
-# Build signed panda firmware
-pushd panda/
-CERT=/tmp/pandaextra/certs/release RELEASE=1 scons -u .
-mv board/obj/panda.bin.signed /tmp/panda.bin.signed
-popd
-
-# Build stuff
-ln -sfn /data/openpilot /data/pythonpath
-export PYTHONPATH="/data/openpilot:/data/openpilot/pyextra"
-scons -j3
-
-# Run tests
-python selfdrive/manager/test/test_manager.py
-selfdrive/car/tests/test_car_interfaces.py
-
-# Ensure no submodules in release
-if test "$(git submodule--helper list | wc -l)" -gt "0"; then
-  echo "submodules found:"
-  git submodule--helper list
-  exit 1
-fi
-git submodule status
-
-# Cleanup
-find . -name '*.a' -delete
-find . -name '*.o' -delete
-find . -name '*.os' -delete
-find . -name '*.pyc' -delete
-find . -name '__pycache__' -delete
-rm -rf panda/board panda/certs panda/crypto
-rm -rf .sconsign.dblite Jenkinsfile release/
-rm models/supercombo.dlc
-
-# Move back signed panda fw
-mkdir -p panda/board/obj
-mv /tmp/panda.bin.signed panda/board/obj/panda.bin.signed
-
-# Restore third_party
-git checkout third_party/
-
-# Mark as prebuilt release
-touch prebuilt
-
-# Add built files to git
-git add -f .
-git commit --amend -m "openpilot v$VERSION"
-
-# Print committed files that are normally gitignored
-#git status --ignored
-
-if [ ! -z "$PUSH" ]; then
-  git remote set-url origin git@github.com:commaai/openpilot.git
-
-  # Push to release2-staging
-  git push -f origin release2-staging
-
-  # Create dashcam release
-  git rm selfdrive/car/*/carcontroller.py
-
-  git commit -m "create dashcam release from release2"
-  git push -f origin release2-staging:dashcam-staging
-fi

release/files_eon

@@ -0,0 +1 @@
+README.md

selfdrive/test/helpers.py

@@ -1,3 +1,4 @@
+import os
 import time
 from functools import wraps
 
@@ -24,6 +25,13 @@ def phone_only(f):
     f(self, *args, **kwargs)
   return wrap
 
+def release_only(f):
+  @wraps(f)
+  def wrap(self, *args, **kwargs):
+    if "RELEASE" not in os.environ:
+      self.skipTest("This test is only for release branches")
+    f(self, *args, **kwargs)
+  return wrap
 
 def with_processes(processes, init_time=0, ignore_stopped=None):
   ignore_stopped = [] if ignore_stopped is None else ignore_stopped

selfdrive/test/test_onroad.py

@@ -8,14 +8,16 @@ import unittest
 from collections import Counter
 from pathlib import Path
 
+from cereal import car
 import cereal.messaging as messaging
 from cereal.services import service_list
 from common.basedir import BASEDIR
 from common.timeout import Timeout
 from common.params import Params
+from selfdrive.controls.lib.events import EVENTS, ET
 from selfdrive.hardware import EON, TICI
 from selfdrive.loggerd.config import ROOT
-from selfdrive.test.helpers import set_params_enabled
+from selfdrive.test.helpers import set_params_enabled, release_only
 from tools.lib.logreader import LogReader
 
 # Baseline CPU usage by process
@@ -56,8 +58,9 @@ if TICI:
     "./camerad": 31.0,
     "./_ui": 21.0,
     "selfdrive.controls.plannerd": 11.7,
-    "selfdrive.locationd.paramsd": 5.0,
     "./_dmonitoringmodeld": 10.0,
+    "selfdrive.locationd.paramsd": 5.0,
+    "selfdrive.controls.radard": 3.6,
     "selfdrive.thermald.thermald": 1.5,
   })
 
@@ -128,7 +131,7 @@ class TestOnroad(unittest.TestCase):
     if "DEBUG" in os.environ:
       segs = filter(lambda x: os.path.exists(os.path.join(x, "rlog.bz2")), Path(ROOT).iterdir())
       segs = sorted(segs, key=lambda x: x.stat().st_mtime)
-      cls.lr = list(LogReader(os.path.join(segs[-2], "rlog.bz2")))
+      cls.lr = list(LogReader(os.path.join(segs[-1], "rlog.bz2")))
       return
 
     os.environ['SKIP_FW_QUERY'] = "1"
@@ -171,6 +174,9 @@ class TestOnroad(unittest.TestCase):
       if proc.wait(60) is None:
         proc.kill()
 
+    cls.lrs = [list(LogReader(os.path.join(str(s), "rlog.bz2"))) for s in cls.segments]
+
+    # use the second segment by default as it's the first full segment
     cls.lr = list(LogReader(os.path.join(str(cls.segments[1]), "rlog.bz2")))
 
   def test_cloudlog_size(self):
@@ -222,5 +228,17 @@ class TestOnroad(unittest.TestCase):
       print(f"     {np.max(np.absolute([np.max(ts)/dt, np.min(ts)/dt]))} {np.std(ts)/dt}")
     print("="*67)
 
+  @release_only
+  def test_startup(self):
+    startup_alert = None
+    for msg in self.lrs[0]:
+      # can't use carEvents because the first msg can be dropped while loggerd is starting up
+      if msg.which() == "controlsState":
+        startup_alert = msg.controlsState.alertText1
+        break
+    expected = EVENTS[car.CarEvent.EventName.startup][ET.PERMANENT].alert_text_1
+    self.assertEqual(startup_alert, expected, "wrong startup alert")
+
+
 if __name__ == "__main__":
   unittest.main()