diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 9e30cc0bb..24278af21 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -244,6 +244,21 @@ jobs: name: process_replay_diff.txt path: selfdrive/test/process_replay/diff.txt + #model_replay: + # name: model replay + # runs-on: ubuntu-20.04 + # timeout-minutes: 50 + # steps: + # - uses: actions/checkout@v2 + # with: + # submodules: true + # - name: Build Docker image + # run: eval "$BUILD" + # - name: Run replay + # run: | + # ${{ env.RUN }} "scons -j$(nproc) && \ + # selfdrive/test/process_replay/model_replay.py" + test_longitudinal: name: longitudinal runs-on: ubuntu-20.04 diff --git a/Jenkinsfile b/Jenkinsfile index 34d78e51f..127051fa8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -138,8 +138,8 @@ pipeline { stage('Replay Tests') { steps { phone_steps("eon2", [ - ["build QCOM_REPLAY", "cd selfdrive/manager && QCOM_REPLAY=1 ./build.py"], - ["camerad/modeld replay", "cd selfdrive/test/process_replay && ./camera_replay.py"], + ["build", "cd selfdrive/manager && ./build.py"], + ["model replay", "cd selfdrive/test/process_replay && ./model_replay.py"], ]) } } diff --git a/SConstruct b/SConstruct index 992e7b57a..d0ddc62b1 100644 --- a/SConstruct +++ b/SConstruct @@ -37,6 +37,10 @@ AddOption('--mpc-generate', action='store_true', help='regenerates the mpc sources') +AddOption('--snpe', + action='store_true', + help='use SNPE on PC') + AddOption('--external-sconscript', action='store', metavar='FILE', @@ -51,7 +55,6 @@ if arch == "aarch64" and TICI: arch = "larch64" USE_WEBCAM = os.getenv("USE_WEBCAM") is not None -QCOM_REPLAY = arch == "aarch64" and os.getenv("QCOM_REPLAY") is not None lenv = { "PATH": os.environ['PATH'], @@ -98,10 +101,6 @@ if arch == "aarch64" or arch == "larch64": cflags = ["-DQCOM", "-mcpu=cortex-a57"] cxxflags = ["-DQCOM", "-mcpu=cortex-a57"] rpath = [] - - if QCOM_REPLAY: - cflags += ["-DQCOM_REPLAY"] - cxxflags += ["-DQCOM_REPLAY"] else: cflags = [] cxxflags = [] @@ -338,7 +337,7 @@ if GetOption("clazy"): qt_env['ENV']['CLAZY_IGNORE_DIRS'] = qt_dirs[0] qt_env['ENV']['CLAZY_CHECKS'] = ','.join(checks) -Export('env', 'qt_env', 'arch', 'real_arch', 'SHARED', 'USE_WEBCAM', 'QCOM_REPLAY') +Export('env', 'qt_env', 'arch', 'real_arch', 'SHARED', 'USE_WEBCAM') # cereal and messaging are shared with the system SConscript(['cereal/SConscript']) diff --git a/cereal b/cereal index 7609d22fb..d87e7e56d 160000 --- a/cereal +++ b/cereal @@ -1 +1 @@ -Subproject commit 7609d22fb2f8ba559a2f8a9e88985dffcfb06d5d +Subproject commit d87e7e56d931413d0625fb765c3142ed6106c47b diff --git a/selfdrive/camerad/SConscript b/selfdrive/camerad/SConscript index a26c20dd5..c12d25407 100644 --- a/selfdrive/camerad/SConscript +++ b/selfdrive/camerad/SConscript @@ -1,13 +1,10 @@ -Import('env', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc', 'USE_WEBCAM', 'QCOM_REPLAY') +Import('env', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc', 'USE_WEBCAM') libs = ['m', 'pthread', common, 'jpeg', 'OpenCL', cereal, messaging, 'zmq', 'capnp', 'kj', visionipc, gpucommon] if arch == "aarch64": libs += ['gsl', 'CB', 'adreno_utils', 'EGL', 'GLESv3', 'cutils', 'ui'] - if QCOM_REPLAY: - cameras = ['cameras/camera_frame_stream.cc'] - else: - cameras = ['cameras/camera_qcom.cc'] + cameras = ['cameras/camera_qcom.cc'] elif arch == "larch64": libs += ['atomic'] cameras = ['cameras/camera_qcom2.cc'] diff --git a/selfdrive/camerad/cameras/camera_common.cc b/selfdrive/camerad/cameras/camera_common.cc index 6294c1580..cf629bde3 100644 --- a/selfdrive/camerad/cameras/camera_common.cc +++ b/selfdrive/camerad/cameras/camera_common.cc @@ -17,7 +17,7 @@ #include "selfdrive/common/util.h" #include "selfdrive/hardware/hw.h" -#if defined(QCOM) && !defined(QCOM_REPLAY) +#ifdef QCOM #include "selfdrive/camerad/cameras/camera_qcom.h" #elif QCOM2 #include "selfdrive/camerad/cameras/camera_qcom2.h" diff --git a/selfdrive/camerad/main.cc b/selfdrive/camerad/main.cc index 6cfa85bdb..9636fd27b 100644 --- a/selfdrive/camerad/main.cc +++ b/selfdrive/camerad/main.cc @@ -15,7 +15,7 @@ #include "selfdrive/common/util.h" #include "selfdrive/hardware/hw.h" -#if defined(QCOM) && !defined(QCOM_REPLAY) +#ifdef QCOM #include "selfdrive/camerad/cameras/camera_qcom.h" #elif QCOM2 #include "selfdrive/camerad/cameras/camera_qcom2.h" diff --git a/selfdrive/common/SConscript b/selfdrive/common/SConscript index 5d5900500..8f6c1dc18 100644 --- a/selfdrive/common/SConscript +++ b/selfdrive/common/SConscript @@ -1,4 +1,4 @@ -Import('env', 'arch', 'SHARED', 'QCOM_REPLAY') +Import('env', 'arch', 'SHARED') if SHARED: fxn = env.SharedLibrary diff --git a/selfdrive/modeld/SConscript b/selfdrive/modeld/SConscript index 19c57f1dd..a0548ee36 100644 --- a/selfdrive/modeld/SConscript +++ b/selfdrive/modeld/SConscript @@ -37,12 +37,13 @@ if arch == "aarch64" or arch == "larch64": else: libs += ['pthread'] - # for onnx support - common_src += ['runners/onnxmodel.cc'] + if not GetOption('snpe'): + # for onnx support + common_src += ['runners/onnxmodel.cc'] - # tell runners to use onnx - lenv['CFLAGS'].append("-DUSE_ONNX_MODEL") - lenv['CXXFLAGS'].append("-DUSE_ONNX_MODEL") + # tell runners to use onnx + lenv['CFLAGS'].append("-DUSE_ONNX_MODEL") + lenv['CXXFLAGS'].append("-DUSE_ONNX_MODEL") if arch == "Darwin": # fix OpenCL @@ -56,10 +57,9 @@ else: common_model = lenv.Object(common_src) - # build thneed model if arch == "aarch64" or arch == "larch64": - compiler = lenv.Program('thneed/compile', ["thneed/compile.cc" ]+common_model, LIBS=libs) + compiler = lenv.Program('thneed/compile', ["thneed/compile.cc"]+common_model, LIBS=libs) cmd = f"cd {Dir('.').abspath} && {compiler[0].abspath} ../../models/supercombo.dlc ../../models/supercombo.thneed --binary" lib_paths = ':'.join([Dir(p).abspath for p in lenv["LIBPATH"]]) diff --git a/selfdrive/modeld/modeld b/selfdrive/modeld/modeld index 668685957..a87221dbd 100755 --- a/selfdrive/modeld/modeld +++ b/selfdrive/modeld/modeld @@ -1,12 +1,15 @@ #!/bin/sh + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" + if [ -d /system ]; then - if [ -f /TICI ]; then # QCOM2 - export LD_LIBRARY_PATH="/usr/lib/aarch64-linux-gnu:/data/pythonpath/phonelibs/snpe/larch64:$LD_LIBRARY_PATH" - else # QCOM - export LD_LIBRARY_PATH="/data/pythonpath/phonelibs/snpe/aarch64/:$LD_LIBRARY_PATH" - fi + if [ -f /TICI ]; then # QCOM2 + export LD_LIBRARY_PATH="/usr/lib/aarch64-linux-gnu:/data/pythonpath/phonelibs/snpe/larch64:$LD_LIBRARY_PATH" + else # QCOM + export LD_LIBRARY_PATH="/data/pythonpath/phonelibs/snpe/aarch64/:$LD_LIBRARY_PATH" + fi else - # PC - export LD_LIBRARY_PATH="$HOME/openpilot/phonelibs/snpe/x86_64-linux-clang:/openpilot/phonelibs/snpe/x86_64:$HOME/openpilot/phonelibs/snpe/x86_64:$LD_LIBRARY_PATH" + # PC + export LD_LIBRARY_PATH="$DIR/../../phonelibs/snpe/x86_64-linux-clang:$DIR/../..//openpilot/phonelibs/snpe/x86_64:$LD_LIBRARY_PATH" fi exec ./_modeld diff --git a/selfdrive/test/ci_shell.sh b/selfdrive/test/ci_shell.sh index bf14875e0..a5ff714b2 100755 --- a/selfdrive/test/ci_shell.sh +++ b/selfdrive/test/ci_shell.sh @@ -12,8 +12,8 @@ fi docker run \ -it \ --rm \ - --volume $OP_ROOT:/tmp/openpilot \ - --workdir /tmp/openpilot \ - --env PYTHONPATH=/tmp/openpilot \ + --volume $OP_ROOT:$OP_ROOT \ + --workdir $PWD \ + --env PYTHONPATH=$OP_ROOT \ ghcr.io/commaai/openpilot-base:latest \ /bin/bash diff --git a/selfdrive/test/model_replay.py b/selfdrive/test/model_test.py similarity index 87% rename from selfdrive/test/model_replay.py rename to selfdrive/test/model_test.py index 1ed3e5814..b8a4ac0a8 100755 --- a/selfdrive/test/model_replay.py +++ b/selfdrive/test/model_test.py @@ -5,7 +5,7 @@ import numpy as np from tools.lib.logreader import LogReader from tools.lib.framereader import FrameReader from tools.lib.cache import cache_path_for_file_path -from selfdrive.test.process_replay.camera_replay import camera_replay +from selfdrive.test.process_replay.model_replay import model_replay if __name__ == "__main__": @@ -15,7 +15,7 @@ if __name__ == "__main__": calib = np.load(os.path.expanduser('~/calib.npy')) try: - msgs = camera_replay(list(lr), fr, desire=desire, calib=calib) + msgs = model_replay(list(lr), fr, desire=desire, calib=calib) finally: cache_path = cache_path_for_file_path(os.path.expanduser('~/fcamera.hevc')) if os.path.isfile(cache_path): diff --git a/selfdrive/test/process_replay/camera_replay.py b/selfdrive/test/process_replay/model_replay.py similarity index 77% rename from selfdrive/test/process_replay/camera_replay.py rename to selfdrive/test/process_replay/model_replay.py index 95631956a..b3c52879c 100755 --- a/selfdrive/test/process_replay/camera_replay.py +++ b/selfdrive/test/process_replay/model_replay.py @@ -8,9 +8,11 @@ from tqdm import tqdm import cereal.messaging as messaging from cereal import log +from cereal.visionipc.visionipc_pyx import VisionIpcServer, VisionStreamType # pylint: disable=no-name-in-module, import-error from common.spinner import Spinner from common.timeout import Timeout -from common.transformations.camera import get_view_frame_from_road_frame +from common.transformations.camera import get_view_frame_from_road_frame, eon_f_frame_size, tici_f_frame_size +from selfdrive.hardware import PC from selfdrive.manager.process_config import managed_processes from selfdrive.test.openpilotci import BASE_URL, get_url from selfdrive.test.process_replay.compare_logs import compare_logs, save_log @@ -29,25 +31,20 @@ def replace_calib(msg, calib): return msg -def camera_replay(lr, fr, desire=None, calib=None): +def model_replay(lr, fr, desire=None, calib=None): spinner = Spinner() spinner.update("starting model replay") + vipc_server = None pm = messaging.PubMaster(['roadCameraState', 'liveCalibration', 'lateralPlan']) sm = messaging.SubMaster(['modelV2']) # TODO: add dmonitoringmodeld - print("preparing procs") - managed_processes['camerad'].prepare() - managed_processes['modeld'].prepare() try: - print("starting procs") - managed_processes['camerad'].start() managed_processes['modeld'].start() time.sleep(5) sm.update(1000) - print("procs started") desires_by_index = {v:k for k,v in log.LateralPlan.Desire.schema.enumerants.items()} @@ -68,26 +65,33 @@ def camera_replay(lr, fr, desire=None, calib=None): pm.send('lateralPlan', dat) f = msg.as_builder() - img = fr.get(frame_idx, pix_fmt="rgb24")[0][:,:,::-1] - f.roadCameraState.image = img.flatten().tobytes() - frame_idx += 1 - pm.send(msg.which(), f) + + img = fr.get(frame_idx, pix_fmt="yuv420p")[0] + if vipc_server is None: + w, h = {int(3*w*h/2): (w, h) for (w, h) in [tici_f_frame_size, eon_f_frame_size]}[len(img)] + vipc_server = VisionIpcServer("camerad") + vipc_server.create_buffers(VisionStreamType.VISION_STREAM_YUV_BACK, 40, False, w, h) + vipc_server.start_listener() + time.sleep(1) # wait for modeld to connect + + vipc_server.send(VisionStreamType.VISION_STREAM_YUV_BACK, img.flatten().tobytes(), f.roadCameraState.frameId, + f.roadCameraState.timestampSof, f.roadCameraState.timestampEof) + with Timeout(seconds=15): log_msgs.append(messaging.recv_one(sm.sock['modelV2'])) spinner.update("modeld replay %d/%d" % (frame_idx, fr.frame_count)) + frame_idx += 1 if frame_idx >= fr.frame_count: break except KeyboardInterrupt: pass + finally: + spinner.close() + managed_processes['modeld'].stop() - print("replay done") - spinner.close() - managed_processes['modeld'].stop() - time.sleep(2) - managed_processes['camerad'].stop() return log_msgs if __name__ == "__main__": @@ -100,7 +104,7 @@ if __name__ == "__main__": lr = LogReader(get_url(TEST_ROUTE, 0)) fr = FrameReader(get_url(TEST_ROUTE, 0, log_type="fcamera")) - log_msgs = camera_replay(list(lr), fr) + log_msgs = model_replay(list(lr), fr) failed = False if not update: @@ -111,8 +115,9 @@ if __name__ == "__main__": ignore = ['logMonoTime', 'valid', 'modelV2.frameDropPerc', 'modelV2.modelExecutionTime'] + tolerance = None if not PC else 1e-3 results: Any = {TEST_ROUTE: {}} - results[TEST_ROUTE]["modeld"] = compare_logs(cmp_log, log_msgs, ignore_fields=ignore) + results[TEST_ROUTE]["modeld"] = compare_logs(cmp_log, log_msgs, tolerance=tolerance, ignore_fields=ignore) diff1, diff2, failed = format_diff(results, ref_commit) print(diff2) diff --git a/selfdrive/test/process_replay/model_replay_ref_commit b/selfdrive/test/process_replay/model_replay_ref_commit index 8c97a7e55..9e0670e42 100644 --- a/selfdrive/test/process_replay/model_replay_ref_commit +++ b/selfdrive/test/process_replay/model_replay_ref_commit @@ -1 +1 @@ -b1447cc1b5492ab28a6fafddf25ade7fc66606bf +8f7ed52c84e9e2e6e8f4d2165130b46f27e76b30 \ No newline at end of file diff --git a/tools/replay/unlogger.py b/tools/replay/unlogger.py index 93dd11407..564bc3f02 100755 --- a/tools/replay/unlogger.py +++ b/tools/replay/unlogger.py @@ -29,7 +29,8 @@ SeekAbsoluteTime = namedtuple("SeekAbsoluteTime", ("secs",)) SeekRelativeTime = namedtuple("SeekRelativeTime", ("secs",)) TogglePause = namedtuple("TogglePause", ()) StopAndQuit = namedtuple("StopAndQuit", ()) -VIPC_TYP = "vipc" +VIPC_RGB = "rgb" +VIPC_YUV = "yuv" class UnloggerWorker(object): @@ -121,9 +122,14 @@ class UnloggerWorker(object): smsg.roadCameraState.image = bts extra = (smsg.roadCameraState.frameId, smsg.roadCameraState.timestampSof, smsg.roadCameraState.timestampEof) - data_socket.send_pyobj((cookie, VIPC_TYP, msg.logMonoTime, route_time, extra), flags=zmq.SNDMORE) + data_socket.send_pyobj((cookie, VIPC_RGB, msg.logMonoTime, route_time, extra), flags=zmq.SNDMORE) data_socket.send(bts, copy=False) + img_yuv = self._frame_reader.get(frame_id, pix_fmt="yuv420p") + if img_yuv is not None: + data_socket.send_pyobj((cookie, VIPC_YUV, msg.logMonoTime, route_time, extra), flags=zmq.SNDMORE) + data_socket.send(img_yuv.flatten().tobytes(), copy=False) + data_socket.send_pyobj((cookie, typ, msg.logMonoTime, route_time), flags=zmq.SNDMORE) data_socket.send(smsg.to_bytes(), copy=False) @@ -166,6 +172,7 @@ def _get_vipc_server(length): vipc_server = VisionIpcServer("camerad") vipc_server.create_buffers(VisionStreamType.VISION_STREAM_RGB_BACK, 4, True, w, h) + vipc_server.create_buffers(VisionStreamType.VISION_STREAM_YUV_BACK, 40, False, w, h) vipc_server.start_listener() return vipc_server @@ -259,7 +266,7 @@ def unlogger_thread(command_address, forward_commands_address, data_address, run print("at", route_time) printed_at = route_time - if typ not in send_funcs and typ != 'vipc': + if typ not in send_funcs and typ not in [VIPC_RGB, VIPC_YUV]: if typ in address_mapping: # Remove so we don't keep printing warnings. address = address_mapping.pop(typ) @@ -288,13 +295,15 @@ def unlogger_thread(command_address, forward_commands_address, data_address, run # Send message. try: - if typ == VIPC_TYP and (not no_visionipc): - if vipc_server is None: - vipc_server = _get_vipc_server(len(msg_bytes)) - - i, sof, eof = extra[0] - vipc_server.send(VisionStreamType.VISION_STREAM_RGB_BACK, msg_bytes, i, sof, eof) - if typ != VIPC_TYP: + if typ in [VIPC_RGB, VIPC_YUV]: + if not no_visionipc: + if vipc_server is None: + vipc_server = _get_vipc_server(len(msg_bytes)) + + i, sof, eof = extra[0] + stream = VisionStreamType.VISION_STREAM_RGB_BACK if typ == VIPC_RGB else VisionStreamType.VISION_STREAM_YUV_BACK + vipc_server.send(stream, msg_bytes, i, sof, eof) + else: send_funcs[typ](msg_bytes) except MultiplePublishersError: del send_funcs[typ]