On-device CI framework (#1784)

* let's see if this works

* fix build_release actions job

* does jenkins like this config

* separate jenkinsfile for release build

* fix devel build

* devel build should work

* always pass that for now

* run modeld replay

* release2 build will be a separate PR

* pass env to phone shell

* force checkout

* run on real jenkins eons

* add timeout

* rsync

* more timeout

* trailing slash

* fix branch detection

* debug

* not sure why paramiko doesn't pass it through

* newline

* CI_PUSH

* still not passing it

* test branch

* should be good now
old-commit-hash: 32f03ec8a5

Jenkinsfile

@@ -4,38 +4,42 @@ pipeline {
       image 'python:3.7.3'
       args '--user=root'
     }
-
   }
   environment {
     COMMA_JWT = credentials('athena-test-jwt')
+    CI_PUSH = "${env.BRANCH_NAME == 'master' ? 'master-ci' : ''}"
   }
+
   stages {
-    stage('Device Tests') {
+    stage('On-device Tests') {
       parallel {
-        stage('Build/Test') {
+
+        stage('Build') {
           steps {
-            lock(resource: "", label: 'eon', inversePrecedence: true, variable: 'eon_name', quantity: 1){
+            lock(resource: "", label: 'eon', inversePrecedence: true, variable: 'eon_ip', quantity: 1){
               timeout(time: 30, unit: 'MINUTES') {
-                dir(path: 'release') {
+                dir(path: 'selfdrive/test') {
                   sh 'pip install paramiko'
-                  sh 'python remote_build.py'
+                  sh 'python phone_ci.py "cd release && ./build_devel.sh"'
                 }
               }
             }
           }
         }
+
         stage('Replay Tests') {
           steps {
-            lock(resource: "", label: 'eon2', inversePrecedence: true, variable: 'eon_name', quantity: 1){
+            lock(resource: "", label: 'eon2', inversePrecedence: true, variable: 'eon_ip', quantity: 1){
-              timeout(time: 45, unit: 'MINUTES') {
+              timeout(time: 60, unit: 'MINUTES') {
                 dir(path: 'selfdrive/test') {
                   sh 'pip install paramiko'
-                  sh 'python phone_ci.py'
+                  sh 'python phone_ci.py "cd selfdrive/test/process_replay && ./camera_replay.py"'
                 }
               }
             }
           }
         }
+
       }
     }
   }

SConstruct

@@ -161,7 +161,10 @@ env = Environment(
 )
 
 if os.environ.get('SCONS_CACHE'):
-  CacheDir('/tmp/scons_cache')
+  if QCOM_REPLAY:
+    CacheDir('/tmp/scons_cache_qcom_replay')
+  else:
+    CacheDir('/tmp/scons_cache')
 
 node_interval = 5
 node_count = 0

release/build_devel.sh

@@ -37,7 +37,7 @@ echo "[-] bringing master-ci and devel in sync T=$SECONDS"
 git fetch origin master-ci
 git fetch origin devel
 
-git checkout --track origin/master-ci || true
+git checkout -f --track origin/master-ci
 git reset --hard master-ci
 git checkout master-ci
 git reset --hard origin/devel
@@ -93,9 +93,9 @@ pushd panda/board/pedal
 make obj/comma.bin
 popd
 
-if [ ! -z "$PUSH" ]; then
+if [ ! -z "$CI_PUSH" ]; then
-  echo "[-] Pushing to $PUSH T=$SECONDS"
+  echo "[-] Pushing to $CI_PUSH T=$SECONDS"
-  git push -f origin master-ci:$PUSH
+  git push -f origin master-ci:$CI_PUSH
 fi
 
 echo "[-] done pushing T=$SECONDS"

release/files_common

@@ -1,15 +1,15 @@
-README.md
-SAFETY.md
-
 .gitignore
 LICENSE
 launch_chffrplus.sh
 launch_openpilot.sh
 
+Jenkinsfile
+SConstruct
+
 CONTRIBUTING.md
+README.md
 RELEASES.md
-
+SAFETY.md
-SConstruct
 
 apk/ai.comma*.apk
 

release/remote_build.py

@@ -1,73 +0,0 @@
-#!/usr/bin/env python2
-import paramiko  # pylint: disable=import-error
-import os
-import sys
-import re
-import time
-import socket
-
-
-def start_build(name):
-  ssh = paramiko.SSHClient()
-  ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
-  key_file = open(os.path.join(os.path.dirname(__file__), "../tools/ssh/key/id_rsa"))
-  key = paramiko.RSAKey.from_private_key(key_file)
-
-  print("SSH to phone {}".format(name))
-
-  # Try connecting for one minute
-  t_start = time.time()
-  while True:
-    try:
-      ssh.connect(hostname=name, port=8022, pkey=key, timeout=10)
-    except (paramiko.ssh_exception.SSHException, socket.timeout, paramiko.ssh_exception.NoValidConnectionsError):
-      print("Connection failed")
-      if time.time() - t_start > 60:
-        raise
-    else:
-      break
-    time.sleep(1)
-
-  conn = ssh.invoke_shell()
-  branch = os.environ['GIT_BRANCH']
-  commit = os.environ.get('GIT_COMMIT', branch)
-
-  conn.send('uname -a\n')
-
-  conn.send('cd /data/openpilot_source\n')
-  conn.send("git reset --hard\n")
-  conn.send("git fetch origin\n")
-  conn.send("git checkout %s\n" % commit)
-  conn.send("git clean -xdf\n")
-  conn.send("git submodule update --init\n")
-  conn.send("git submodule foreach --recursive git reset --hard\n")
-  conn.send("git submodule foreach --recursive git clean -xdf\n")
-  conn.send("echo \"git took $SECONDS seconds\"\n")
-
-  push = "PUSH=master-ci" if branch == "master" else ""
-
-  conn.send("%s /data/openpilot_source/release/build_devel.sh\n" % push)
-  conn.send('echo "RESULT:" $?\n')
-  conn.send("exit\n")
-  return conn
-
-
-if __name__ == "__main__":
-  eon_name = os.environ.get('eon_name', None)
-
-  conn = start_build(eon_name)
-
-  dat = b""
-
-  while True:
-    recvd = conn.recv(4096)
-    if len(recvd) == 0:
-      break
-
-    dat += recvd
-    sys.stdout.buffer.write(recvd)
-    sys.stdout.flush()
-
-  returns = re.findall(rb'^RESULT: (\d+)', dat[-1024:], flags=re.MULTILINE)
-  sys.exit(int(returns[0]))

selfdrive/test/phone_ci.py

@@ -6,22 +6,29 @@ import re
 import time
 import socket
 
-TEST_DIR = "/data/openpilotci"
 
-def run_test(name, test_func):
+SOURCE_DIR = "/data/openpilot_source/"
+TEST_DIR = "/data/openpilot/"
+
+def run_on_phone(test_cmd):
+
+  eon_ip = os.environ.get('eon_ip', None)
+  if eon_ip is None:
+    raise Exception("'eon_ip' not set")
+
   ssh = paramiko.SSHClient()
   ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
   key_file = open(os.path.join(os.path.dirname(__file__), "../../tools/ssh/key/id_rsa"))
   key = paramiko.RSAKey.from_private_key(key_file)
 
-  print("SSH to phone {}".format(name))
+  print("SSH to phone at {}".format(eon_ip))
 
-  # Try connecting for one minute
+  # try connecting for one minute
   t_start = time.time()
   while True:
     try:
-      ssh.connect(hostname=name, port=8022, pkey=key, timeout=10)
+      ssh.connect(hostname=eon_ip, port=8022, pkey=key, timeout=10)
     except (paramiko.ssh_exception.SSHException, socket.timeout, paramiko.ssh_exception.NoValidConnectionsError):
       print("Connection failed")
       if time.time() - t_start > 60:
@@ -30,40 +37,47 @@ def run_test(name, test_func):
       break
     time.sleep(1)
 
-  conn = ssh.invoke_shell()
   branch = os.environ['GIT_BRANCH']
   commit = os.environ.get('GIT_COMMIT', branch)
 
-  conn.send("uname -a\n")
+  conn = ssh.invoke_shell()
+
+  # pass in all environment variables prefixed with 'CI_'
+  for k, v in os.environ.items():
+    if k.startswith("CI_"):
+      conn.send(f"export {k}='{v}'\n")
+  conn.send("export CI=1\n")
 
-  conn.send(f"cd {TEST_DIR}\n")
+  # set up environment
+  conn.send(f"cd {SOURCE_DIR}\n")
   conn.send("git reset --hard\n")
   conn.send("git fetch origin\n")
-  conn.send("git checkout %s\n" % commit)
+  conn.send(f"git checkout {commit}\n")
   conn.send("git clean -xdf\n")
   conn.send("git submodule update --init\n")
   conn.send("git submodule foreach --recursive git reset --hard\n")
   conn.send("git submodule foreach --recursive git clean -xdf\n")
-  conn.send("echo \"git took $SECONDS seconds\"\n")
+  conn.send('echo "git took $SECONDS seconds"\n')
 
-  test_func(conn)
+  conn.send(f"rsync -a --delete {SOURCE_DIR} {TEST_DIR}\n")
 
+  # run the test
+  conn.send(test_cmd + "\n")
+
+  # get the result and print it back out
   conn.send('echo "RESULT:" $?\n')
   conn.send("exit\n")
-  return conn
-
-def test_modeld(conn):
-  conn.send(f"cd selfdrive/test/process_replay && PYTHONPATH={TEST_DIR} ./camera_replay.py\n")
-
-if __name__ == "__main__":
-  eon_name = os.environ.get('eon_name', None)
-
-  conn = run_test(eon_name, test_modeld)
 
   dat = b""
+  conn.settimeout(120)
 
   while True:
-    recvd = conn.recv(4096)
+    try:
+      recvd = conn.recv(4096)
+    except socket.timeout:
+      print("connection to phone timed out")
+      sys.exit(1)
+
     if len(recvd) == 0:
       break
 
@@ -71,5 +85,9 @@ if __name__ == "__main__":
     sys.stdout.buffer.write(recvd)
     sys.stdout.flush()
 
-  returns = re.findall(rb'^RESULT: (\d+)', dat[-1024:], flags=re.MULTILINE)
+  return_code = int(re.findall(rb'^RESULT: (\d+)', dat[-1024:], flags=re.MULTILINE)[0])
-  sys.exit(int(returns[0]))
+  sys.exit(return_code)
+
+
+if __name__ == "__main__":
+  run_on_phone(sys.argv[1])