fix up model tests + tools (#21071)

* unlogger: send yuv stream

* fix up model test tools

* fix unlogger

* rename model replay

* bump cereal

* test in actions

* no ci for now

.github/workflows/selfdrive_tests.yaml

@@ -244,6 +244,21 @@ jobs:
         name: process_replay_diff.txt
         path: selfdrive/test/process_replay/diff.txt
 
+  #model_replay:
+  #  name: model replay
+  #  runs-on: ubuntu-20.04
+  #  timeout-minutes: 50
+  #  steps:
+  #  - uses: actions/checkout@v2
+  #    with:
+  #      submodules: true
+  #  - name: Build Docker image
+  #    run: eval "$BUILD"
+  #  - name: Run replay
+  #    run: |
+  #      ${{ env.RUN }} "scons -j$(nproc) && \
+  #                      selfdrive/test/process_replay/model_replay.py"
+
   test_longitudinal:
     name: longitudinal
     runs-on: ubuntu-20.04

Jenkinsfile

@@ -138,8 +138,8 @@ pipeline {
                 stage('Replay Tests') {
                   steps {
                     phone_steps("eon2", [
-                      ["build QCOM_REPLAY", "cd selfdrive/manager && QCOM_REPLAY=1 ./build.py"],
-                      ["camerad/modeld replay", "cd selfdrive/test/process_replay && ./camera_replay.py"],
+                      ["build", "cd selfdrive/manager && ./build.py"],
+                      ["model replay", "cd selfdrive/test/process_replay && ./model_replay.py"],
                     ])
                   }
                 }

SConstruct

@@ -37,6 +37,10 @@ AddOption('--mpc-generate',
           action='store_true',
           help='regenerates the mpc sources')
 
+AddOption('--snpe',
+          action='store_true',
+          help='use SNPE on PC')
+
 AddOption('--external-sconscript',
           action='store',
           metavar='FILE',
@@ -51,7 +55,6 @@ if arch == "aarch64" and TICI:
   arch = "larch64"
 
 USE_WEBCAM = os.getenv("USE_WEBCAM") is not None
-QCOM_REPLAY = arch == "aarch64" and os.getenv("QCOM_REPLAY") is not None
 
 lenv = {
   "PATH": os.environ['PATH'],
@@ -98,10 +101,6 @@ if arch == "aarch64" or arch == "larch64":
     cflags = ["-DQCOM", "-mcpu=cortex-a57"]
     cxxflags = ["-DQCOM", "-mcpu=cortex-a57"]
     rpath = []
-
-    if QCOM_REPLAY:
-      cflags += ["-DQCOM_REPLAY"]
-      cxxflags += ["-DQCOM_REPLAY"]
 else:
   cflags = []
   cxxflags = []
@@ -338,7 +337,7 @@ if GetOption("clazy"):
   qt_env['ENV']['CLAZY_IGNORE_DIRS'] = qt_dirs[0]
   qt_env['ENV']['CLAZY_CHECKS'] = ','.join(checks)
 
-Export('env', 'qt_env', 'arch', 'real_arch', 'SHARED', 'USE_WEBCAM', 'QCOM_REPLAY')
+Export('env', 'qt_env', 'arch', 'real_arch', 'SHARED', 'USE_WEBCAM')
 
 # cereal and messaging are shared with the system
 SConscript(['cereal/SConscript'])

cereal

@@ -1 +1 @@
-Subproject commit 7609d22fb2f8ba559a2f8a9e88985dffcfb06d5d
+Subproject commit d87e7e56d931413d0625fb765c3142ed6106c47b

selfdrive/camerad/SConscript

@@ -1,12 +1,9 @@
-Import('env', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc', 'USE_WEBCAM', 'QCOM_REPLAY')
+Import('env', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc', 'USE_WEBCAM')
 
 libs = ['m', 'pthread', common, 'jpeg', 'OpenCL', cereal, messaging, 'zmq', 'capnp', 'kj', visionipc, gpucommon]
 
 if arch == "aarch64":
   libs += ['gsl', 'CB', 'adreno_utils', 'EGL', 'GLESv3', 'cutils', 'ui']
-  if QCOM_REPLAY:
-    cameras = ['cameras/camera_frame_stream.cc']
-  else:
   cameras = ['cameras/camera_qcom.cc']
 elif arch == "larch64":
   libs += ['atomic']

selfdrive/camerad/cameras/camera_common.cc

@@ -17,7 +17,7 @@
 #include "selfdrive/common/util.h"
 #include "selfdrive/hardware/hw.h"
 
-#if defined(QCOM) && !defined(QCOM_REPLAY)
+#ifdef QCOM
 #include "selfdrive/camerad/cameras/camera_qcom.h"
 #elif QCOM2
 #include "selfdrive/camerad/cameras/camera_qcom2.h"

selfdrive/camerad/main.cc

@@ -15,7 +15,7 @@
 #include "selfdrive/common/util.h"
 #include "selfdrive/hardware/hw.h"
 
-#if defined(QCOM) && !defined(QCOM_REPLAY)
+#ifdef QCOM
 #include "selfdrive/camerad/cameras/camera_qcom.h"
 #elif QCOM2
 #include "selfdrive/camerad/cameras/camera_qcom2.h"

selfdrive/common/SConscript

@@ -1,4 +1,4 @@
-Import('env', 'arch', 'SHARED', 'QCOM_REPLAY')
+Import('env', 'arch', 'SHARED')
 
 if SHARED:
   fxn = env.SharedLibrary

selfdrive/modeld/SConscript

@@ -37,6 +37,7 @@ if arch == "aarch64" or arch == "larch64":
 else:
   libs += ['pthread']
 
+  if not GetOption('snpe'):
     # for onnx support
     common_src += ['runners/onnxmodel.cc']
 
@@ -56,10 +57,9 @@ else:
 
 common_model = lenv.Object(common_src)
 
-
 # build thneed model
 if arch == "aarch64" or arch == "larch64":
-  compiler = lenv.Program('thneed/compile', ["thneed/compile.cc" ]+common_model, LIBS=libs)
+  compiler = lenv.Program('thneed/compile', ["thneed/compile.cc"]+common_model, LIBS=libs)
   cmd = f"cd {Dir('.').abspath} && {compiler[0].abspath} ../../models/supercombo.dlc ../../models/supercombo.thneed --binary"
 
   lib_paths = ':'.join([Dir(p).abspath for p in lenv["LIBPATH"]])

selfdrive/modeld/modeld

@@ -1,4 +1,7 @@
 #!/bin/sh
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+
 if [ -d /system ]; then
   if [ -f /TICI ]; then # QCOM2
     export LD_LIBRARY_PATH="/usr/lib/aarch64-linux-gnu:/data/pythonpath/phonelibs/snpe/larch64:$LD_LIBRARY_PATH"
@@ -7,6 +10,6 @@ if [ -d /system ]; then
   fi
 else
   # PC
-    export LD_LIBRARY_PATH="$HOME/openpilot/phonelibs/snpe/x86_64-linux-clang:/openpilot/phonelibs/snpe/x86_64:$HOME/openpilot/phonelibs/snpe/x86_64:$LD_LIBRARY_PATH"
+  export LD_LIBRARY_PATH="$DIR/../../phonelibs/snpe/x86_64-linux-clang:$DIR/../..//openpilot/phonelibs/snpe/x86_64:$LD_LIBRARY_PATH"
 fi
 exec ./_modeld

selfdrive/test/ci_shell.sh

@@ -12,8 +12,8 @@ fi
 docker run \
        -it \
        --rm \
-       --volume $OP_ROOT:/tmp/openpilot \
-       --workdir /tmp/openpilot \
-       --env PYTHONPATH=/tmp/openpilot \
+       --volume $OP_ROOT:$OP_ROOT \
+       --workdir $PWD \
+       --env PYTHONPATH=$OP_ROOT \
        ghcr.io/commaai/openpilot-base:latest \
        /bin/bash

selfdrive/test/model_test.py

@@ -5,7 +5,7 @@ import numpy as np
 from tools.lib.logreader import LogReader
 from tools.lib.framereader import FrameReader
 from tools.lib.cache import cache_path_for_file_path
-from selfdrive.test.process_replay.camera_replay import camera_replay
+from selfdrive.test.process_replay.model_replay import model_replay
 
 
 if __name__ == "__main__":
@@ -15,7 +15,7 @@ if __name__ == "__main__":
   calib = np.load(os.path.expanduser('~/calib.npy'))
 
   try:
-    msgs = camera_replay(list(lr), fr, desire=desire, calib=calib)
+    msgs = model_replay(list(lr), fr, desire=desire, calib=calib)
   finally:
     cache_path = cache_path_for_file_path(os.path.expanduser('~/fcamera.hevc'))
     if os.path.isfile(cache_path):

selfdrive/test/process_replay/model_replay.py

@@ -8,9 +8,11 @@ from tqdm import tqdm
 
 import cereal.messaging as messaging
 from cereal import log
+from cereal.visionipc.visionipc_pyx import VisionIpcServer, VisionStreamType  # pylint: disable=no-name-in-module, import-error
 from common.spinner import Spinner
 from common.timeout import Timeout
-from common.transformations.camera import get_view_frame_from_road_frame
+from common.transformations.camera import get_view_frame_from_road_frame, eon_f_frame_size, tici_f_frame_size
+from selfdrive.hardware import PC
 from selfdrive.manager.process_config import managed_processes
 from selfdrive.test.openpilotci import BASE_URL, get_url
 from selfdrive.test.process_replay.compare_logs import compare_logs, save_log
@@ -29,25 +31,20 @@ def replace_calib(msg, calib):
   return msg
 
 
-def camera_replay(lr, fr, desire=None, calib=None):
+def model_replay(lr, fr, desire=None, calib=None):
 
   spinner = Spinner()
   spinner.update("starting model replay")
 
+  vipc_server = None
   pm = messaging.PubMaster(['roadCameraState', 'liveCalibration', 'lateralPlan'])
   sm = messaging.SubMaster(['modelV2'])
 
   # TODO: add dmonitoringmodeld
-  print("preparing procs")
-  managed_processes['camerad'].prepare()
-  managed_processes['modeld'].prepare()
   try:
-    print("starting procs")
-    managed_processes['camerad'].start()
     managed_processes['modeld'].start()
     time.sleep(5)
     sm.update(1000)
-    print("procs started")
 
     desires_by_index = {v:k for k,v in log.LateralPlan.Desire.schema.enumerants.items()}
 
@@ -68,26 +65,33 @@ def camera_replay(lr, fr, desire=None, calib=None):
             pm.send('lateralPlan', dat)
 
         f = msg.as_builder()
-        img = fr.get(frame_idx, pix_fmt="rgb24")[0][:,:,::-1]
-        f.roadCameraState.image = img.flatten().tobytes()
-        frame_idx += 1
-
         pm.send(msg.which(), f)
+
+        img = fr.get(frame_idx, pix_fmt="yuv420p")[0]
+        if vipc_server is None:
+          w, h = {int(3*w*h/2): (w, h) for (w, h) in [tici_f_frame_size, eon_f_frame_size]}[len(img)]
+          vipc_server = VisionIpcServer("camerad")
+          vipc_server.create_buffers(VisionStreamType.VISION_STREAM_YUV_BACK, 40, False, w, h)
+          vipc_server.start_listener()
+          time.sleep(1) # wait for modeld to connect
+
+        vipc_server.send(VisionStreamType.VISION_STREAM_YUV_BACK, img.flatten().tobytes(), f.roadCameraState.frameId,
+                         f.roadCameraState.timestampSof, f.roadCameraState.timestampEof)
+
         with Timeout(seconds=15):
           log_msgs.append(messaging.recv_one(sm.sock['modelV2']))
 
         spinner.update("modeld replay %d/%d" % (frame_idx, fr.frame_count))
 
+        frame_idx += 1
         if frame_idx >= fr.frame_count:
           break
   except KeyboardInterrupt:
     pass
-
-  print("replay done")
+  finally:
     spinner.close()
     managed_processes['modeld'].stop()
-  time.sleep(2)
-  managed_processes['camerad'].stop()
+
   return log_msgs
 
 if __name__ == "__main__":
@@ -100,7 +104,7 @@ if __name__ == "__main__":
   lr = LogReader(get_url(TEST_ROUTE, 0))
   fr = FrameReader(get_url(TEST_ROUTE, 0, log_type="fcamera"))
 
-  log_msgs = camera_replay(list(lr), fr)
+  log_msgs = model_replay(list(lr), fr)
 
   failed = False
   if not update:
@@ -111,8 +115,9 @@ if __name__ == "__main__":
     ignore = ['logMonoTime', 'valid',
               'modelV2.frameDropPerc',
               'modelV2.modelExecutionTime']
+    tolerance = None if not PC else 1e-3
     results: Any = {TEST_ROUTE: {}}
-    results[TEST_ROUTE]["modeld"] = compare_logs(cmp_log, log_msgs, ignore_fields=ignore)
+    results[TEST_ROUTE]["modeld"] = compare_logs(cmp_log, log_msgs, tolerance=tolerance, ignore_fields=ignore)
     diff1, diff2, failed = format_diff(results, ref_commit)
 
     print(diff2)

selfdrive/test/process_replay/model_replay_ref_commit

@@ -1 +1 @@
-b1447cc1b5492ab28a6fafddf25ade7fc66606bf
+8f7ed52c84e9e2e6e8f4d2165130b46f27e76b30

tools/replay/unlogger.py

@@ -29,7 +29,8 @@ SeekAbsoluteTime = namedtuple("SeekAbsoluteTime", ("secs",))
 SeekRelativeTime = namedtuple("SeekRelativeTime", ("secs",))
 TogglePause = namedtuple("TogglePause", ())
 StopAndQuit = namedtuple("StopAndQuit", ())
-VIPC_TYP = "vipc"
+VIPC_RGB = "rgb"
+VIPC_YUV = "yuv"
 
 
 class UnloggerWorker(object):
@@ -121,9 +122,14 @@ class UnloggerWorker(object):
           smsg.roadCameraState.image = bts
 
           extra = (smsg.roadCameraState.frameId, smsg.roadCameraState.timestampSof, smsg.roadCameraState.timestampEof)
-          data_socket.send_pyobj((cookie, VIPC_TYP, msg.logMonoTime, route_time, extra), flags=zmq.SNDMORE)
+          data_socket.send_pyobj((cookie, VIPC_RGB, msg.logMonoTime, route_time, extra), flags=zmq.SNDMORE)
           data_socket.send(bts, copy=False)
 
+          img_yuv = self._frame_reader.get(frame_id, pix_fmt="yuv420p")
+          if img_yuv is not None:
+            data_socket.send_pyobj((cookie, VIPC_YUV, msg.logMonoTime, route_time, extra), flags=zmq.SNDMORE)
+            data_socket.send(img_yuv.flatten().tobytes(), copy=False)
+
       data_socket.send_pyobj((cookie, typ, msg.logMonoTime, route_time), flags=zmq.SNDMORE)
       data_socket.send(smsg.to_bytes(), copy=False)
 
@@ -166,6 +172,7 @@ def _get_vipc_server(length):
 
   vipc_server = VisionIpcServer("camerad")
   vipc_server.create_buffers(VisionStreamType.VISION_STREAM_RGB_BACK, 4, True, w, h)
+  vipc_server.create_buffers(VisionStreamType.VISION_STREAM_YUV_BACK, 40, False, w, h)
   vipc_server.start_listener()
   return vipc_server
 
@@ -259,7 +266,7 @@ def unlogger_thread(command_address, forward_commands_address, data_address, run
         print("at", route_time)
         printed_at = route_time
 
-      if typ not in send_funcs and typ != 'vipc':
+      if typ not in send_funcs and typ not in [VIPC_RGB, VIPC_YUV]:
         if typ in address_mapping:
           # Remove so we don't keep printing warnings.
           address = address_mapping.pop(typ)
@@ -288,13 +295,15 @@ def unlogger_thread(command_address, forward_commands_address, data_address, run
 
       # Send message.
       try:
-        if typ == VIPC_TYP and (not no_visionipc):
+        if typ in [VIPC_RGB, VIPC_YUV]:
+          if not no_visionipc:
             if vipc_server is None:
               vipc_server = _get_vipc_server(len(msg_bytes))
 
             i, sof, eof = extra[0]
-          vipc_server.send(VisionStreamType.VISION_STREAM_RGB_BACK, msg_bytes, i, sof, eof)
-        if typ != VIPC_TYP:
+            stream = VisionStreamType.VISION_STREAM_RGB_BACK if typ == VIPC_RGB else VisionStreamType.VISION_STREAM_YUV_BACK
+            vipc_server.send(stream, msg_bytes, i, sof, eof)
+        else:
           send_funcs[typ](msg_bytes)
       except MultiplePublishersError:
         del send_funcs[typ]