simulator: run simulator test in ci (#24691)

* run simulator test in ci

* block navd process

* block ui

* fix jenkins

* build docker

* remove tty

* remove tty for carla

* detach carla_sim

* more retries

* only build once

* add more time for bridge

* cleanup

* use qt offscreen

* expose to docker

* block ui

* use new dockerimage

* fix

* from ubuntu20.04

* install curl

* add ssh

* add locales

* noninteractive

* syntax

* use base

* smaller image

* add git + git lfs

* kill carla

* run in parallel

* fix missing agents

* default agent?

* little cleanup

* default doesn't work

* not in ci

* fix path

* fix path

* new msg

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>
old-commit-hash: 5add0c6159

Jenkinsfile

@@ -42,6 +42,7 @@ def phone_steps(String device_type, steps) {
 pipeline {
   agent none
   environment {
+    CI = "1"
     TEST_DIR = "/data/openpilot"
     SOURCE_DIR = "/data/openpilot_source/"
   }
@@ -74,71 +75,89 @@ pipeline {
         }
       }
 
-      stages {
-        stage('On-device Tests') {
-          agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } }
-          stages {
-            stage('parallel tests') {
-              parallel {
-                stage('build') {
-                  environment {
-                    R3_PUSH = "${env.BRANCH_NAME == 'master' ? '1' : ' '}"
-                  }
-                  steps {
-                    phone_steps("tici", [
-                      ["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR EXTRA_FILES='tools/' ./build_devel.sh"],
-                      ["build openpilot", "cd selfdrive/manager && ./build.py"],
-                      ["test manager", "python selfdrive/manager/test/test_manager.py"],
-                      ["onroad tests", "cd selfdrive/test/ && ./test_onroad.py"],
-                      ["test car interfaces", "cd selfdrive/car/tests/ && ./test_car_interfaces.py"],
-                    ])
-                  }
-                }
-
-                stage('HW + Unit Tests') {
-                  steps {
-                    phone_steps("tici2", [
-                      ["build", "cd selfdrive/manager && ./build.py"],
-                      ["test power draw", "python selfdrive/hardware/tici/test_power_draw.py"],
-                      ["test boardd loopback", "python selfdrive/boardd/tests/test_boardd_loopback.py"],
-                      ["test loggerd", "python selfdrive/loggerd/tests/test_loggerd.py"],
-                      ["test encoder", "LD_LIBRARY_PATH=/usr/local/lib python selfdrive/loggerd/tests/test_encoder.py"],
-                      ["test sensord", "python selfdrive/sensord/test/test_sensord.py"],
-                    ])
-                  }
-                }
-
-                stage('camerad') {
-                  steps {
-                    phone_steps("tici-party", [
-                      ["build", "cd selfdrive/manager && ./build.py"],
-                      ["test camerad", "python selfdrive/camerad/test/test_camerad.py"],
-                      ["test exposure", "python selfdrive/camerad/test/test_exposure.py"],
-                    ])
-                  }
-                }
-
-                stage('replay') {
-                  steps {
-                    phone_steps("tici3", [
-                      ["build", "cd selfdrive/manager && ./build.py"],
-                      ["model replay", "cd selfdrive/test/process_replay && ./model_replay.py"],
-                    ])
-                  }
-                }
-
-              }
+      parallel {
+
+        stage('simulator') {
+          agent {
+            dockerfile {
+              filename 'Dockerfile.sim_nvidia'
+              dir 'tools/sim'
+              args '--user=root'
             }
           }
+          steps {
+            sh "git config --global --add safe.directory ${WORKSPACE}"
+            sh "git lfs pull"
+            sh "${WORKSPACE}/tools/sim/build_container.sh"
+            sh "DETACH=1 ${WORKSPACE}/tools/sim/start_carla.sh"
+            sh "${WORKSPACE}/tools/sim/start_openpilot_docker.sh"
+          }
 
           post {
             always {
-              cleanWs()
+              sh "docker kill carla_sim || true"
+              sh "rm -rf ${WORKSPACE}/* || true"
+              sh "rm -rf .* || true"
             }
           }
+        }
 
+        stage('build') {
+          agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } }
+          environment {
+            R3_PUSH = "${env.BRANCH_NAME == 'master' ? '1' : ' '}"
+          }
+          steps {
+            phone_steps("tici", [
+              ["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR EXTRA_FILES='tools/' ./build_devel.sh"],
+              ["build openpilot", "cd selfdrive/manager && ./build.py"],
+              ["test manager", "python selfdrive/manager/test/test_manager.py"],
+              ["onroad tests", "cd selfdrive/test/ && ./test_onroad.py"],
+              ["test car interfaces", "cd selfdrive/car/tests/ && ./test_car_interfaces.py"],
+            ])
+          }
+        }
+
+        stage('HW + Unit Tests') {
+          agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } }
+          steps {
+            phone_steps("tici2", [
+              ["build", "cd selfdrive/manager && ./build.py"],
+              ["test power draw", "python selfdrive/hardware/tici/test_power_draw.py"],
+              ["test boardd loopback", "python selfdrive/boardd/tests/test_boardd_loopback.py"],
+              ["test loggerd", "python selfdrive/loggerd/tests/test_loggerd.py"],
+              ["test encoder", "LD_LIBRARY_PATH=/usr/local/lib python selfdrive/loggerd/tests/test_encoder.py"],
+              ["test sensord", "python selfdrive/sensord/test/test_sensord.py"],
+            ])
+          }
         }
 
+        stage('camerad') {
+          agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } }
+          steps {
+            phone_steps("tici-party", [
+              ["build", "cd selfdrive/manager && ./build.py"],
+              ["test camerad", "python selfdrive/camerad/test/test_camerad.py"],
+              ["test exposure", "python selfdrive/camerad/test/test_exposure.py"],
+            ])
+          }
+        }
+
+        stage('replay') {
+          agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } }
+          steps {
+            phone_steps("tici3", [
+              ["build", "cd selfdrive/manager && ./build.py"],
+              ["model replay", "cd selfdrive/test/process_replay && ./model_replay.py"],
+            ])
+          }
+        }
+      }
+
+      post {
+        always {
+          cleanWs()
+        }
       }
     }
   }

tools/sim/Dockerfile.sim

@@ -27,6 +27,6 @@ COPY ./system $HOME/openpilot/system
 COPY ./tools $HOME/openpilot/tools
 
 WORKDIR $HOME/openpilot
-RUN scons -j$(nproc)
+RUN scons -j12
 
 RUN python -c "from selfdrive.test.helpers import set_params_enabled; set_params_enabled()"

tools/sim/Dockerfile.sim_nvidia

@@ -0,0 +1,21 @@
+FROM ubuntu:20.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+      apt-utils \
+      sudo \
+      ssh \
+      curl \
+      ca-certificates \
+      git \
+      git-lfs && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN curl -fsSL https://get.docker.com -o get-docker.sh && \
+    sudo sh get-docker.sh && \
+    distribution=$(. /etc/os-release;echo $ID$VERSION_ID) && \
+    curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - && \
+    curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list && \
+    sudo apt-get update && \
+    sudo apt-get install -y nvidia-docker2

tools/sim/launch_openpilot.sh

@@ -6,6 +6,10 @@ export SIMULATION="1"
 export FINGERPRINT="HONDA CIVIC 2016"
 
 export BLOCK="camerad,loggerd,encoderd"
+if [[ "$CI" ]]; then
+  # TODO: offscreen UI should work
+  export BLOCK="${BLOCK},ui"
+fi
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 cd ../../selfdrive/manager && exec ./manager.py

tools/sim/lib/can.py

@@ -63,6 +63,7 @@ def can_function(pm, speed, angle, idx, cruise_button, is_engaged):
   msg.append(packer.make_can_msg("SCM_FEEDBACK", 0, {"MAIN_ON": 1}, idx))
   msg.append(packer.make_can_msg("POWERTRAIN_DATA", 0, {"ACC_STATUS": int(is_engaged)}, idx))
   msg.append(packer.make_can_msg("HUD_SETTING", 0, {}))
+  msg.append(packer.make_can_msg("CAR_SPEED", 0, {}))
 
   # *** cam bus ***
   msg.append(packer.make_can_msg("STEERING_CONTROL", 2, {}, idx))

tools/sim/start_carla.sh

@@ -17,12 +17,17 @@ fi
 
 docker pull carlasim/carla:0.9.12
 
+EXTRA_ARGS="-it"
+if [[ "$DETACH" ]]; then
+  EXTRA_ARGS="-d"
+fi
+
 docker run \
   --name carla_sim \
   --rm \
   --gpus all \
   --net=host \
   -v /tmp/.X11-unix:/tmp/.X11-unix:rw \
-  -it \
+  $EXTRA_ARGS \
   carlasim/carla:0.9.12 \
   /bin/bash ./CarlaUE4.sh -opengl -nosound -RenderOffScreen -benchmark -fps=20 -quality-level=Low

tools/sim/start_openpilot_docker.sh

@@ -3,22 +3,26 @@
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
 cd $DIR
 
-# expose X to the container
-xhost +local:root
-
-docker pull ghcr.io/commaai/openpilot-sim:latest
-
 OPENPILOT_DIR="/openpilot"
-if ! [[ -z "$MOUNT_OPENPILOT" ]]
-then
+if ! [[ -z "$MOUNT_OPENPILOT" ]]; then
   OPENPILOT_DIR="$(dirname $(dirname $DIR))"
   EXTRA_ARGS="-v $OPENPILOT_DIR:$OPENPILOT_DIR -e PYTHONPATH=$OPENPILOT_DIR:$PYTHONPATH"
 fi
 
+if [[ "$CI" ]]; then
+  CMD="CI=1 ${OPENPILOT_DIR}/tools/sim/tests/test_carla_integration.py"
+else
+  # expose X to the container
+  xhost +local:root
+
+  docker pull ghcr.io/commaai/openpilot-sim:latest
+  CMD="./tmux_script.sh $*"
+  EXTRA_ARGS="${EXTRA_ARGS} -it"
+fi
+
 docker run --net=host\
   --name openpilot_client \
   --rm \
-  -it \
   --gpus all \
   --device=/dev/dri:/dev/dri \
   --device=/dev/input:/dev/input \
@@ -29,4 +33,4 @@ docker run --net=host\
   -w "$OPENPILOT_DIR/tools/sim" \
   $EXTRA_ARGS \
   ghcr.io/commaai/openpilot-sim:latest \
-  /bin/bash -c "./tmux_script.sh $*"
+  /bin/bash -c "$CMD"

tools/sim/tests/test_carla_integration.py

@@ -2,13 +2,18 @@
 import subprocess
 import time
 import unittest
+import os
 from multiprocessing import Queue
 
 from cereal import messaging
+from common.basedir import BASEDIR
 from selfdrive.manager.helpers import unblock_stdout
 from tools.sim import bridge
 from tools.sim.bridge import CarlaBridge
 
+CI = "CI" in os.environ
+
+SIM_DIR = os.path.join(BASEDIR, "tools/sim")
 
 class TestCarlaIntegration(unittest.TestCase):
   """
@@ -19,10 +24,12 @@ class TestCarlaIntegration(unittest.TestCase):
 
   def setUp(self):
     self.processes = []
-    # We want to make sure that carla_sim docker isn't still running.
-    subprocess.run("docker rm -f carla_sim", shell=True, stderr=subprocess.PIPE, check=False)
 
-    self.carla_process = subprocess.Popen(".././start_carla.sh")
+    if not CI:
+      # We want to make sure that carla_sim docker isn't still running.
+      subprocess.run("docker rm -f carla_sim", shell=True, stderr=subprocess.PIPE, check=False)
+      self.carla_process = subprocess.Popen("./start_carla.sh", cwd=SIM_DIR)
+
     # Too many lagging messages in bridge.py can cause a crash. This prevents it.
     unblock_stdout()
     # Wait 10 seconds to startup carla
@@ -30,16 +37,16 @@ class TestCarlaIntegration(unittest.TestCase):
 
   def test_engage(self):
     # Startup manager and bridge.py. Check processes are running, then engage and verify.
-    p_manager = subprocess.Popen("./launch_openpilot.sh", cwd='../')
+    p_manager = subprocess.Popen("./launch_openpilot.sh", cwd=SIM_DIR)
     self.processes.append(p_manager)
 
     sm = messaging.SubMaster(['controlsState', 'carEvents', 'managerState'])
     q = Queue()
     carla_bridge = CarlaBridge(bridge.parse_args([]))
-    p_bridge = carla_bridge.run(q, retries=3)
+    p_bridge = carla_bridge.run(q, retries=10)
     self.processes.append(p_bridge)
 
-    max_time_per_step = 20
+    max_time_per_step = 60
 
     # Wait for bridge to startup
     start_waiting = time.monotonic()