diff --git a/.gitattributes b/.gitattributes index eda2505d0e..8781a7371f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,6 +2,7 @@ # to move existing files into LFS: # git add --renormalize . +*.dlc filter=lfs diff=lfs merge=lfs -text *.onnx filter=lfs diff=lfs merge=lfs -text *.svg filter=lfs diff=lfs merge=lfs -text *.png filter=lfs diff=lfs merge=lfs -text diff --git a/selfdrive/modeld/SConscript b/selfdrive/modeld/SConscript index d472998416..d572915c72 100644 --- a/selfdrive/modeld/SConscript +++ b/selfdrive/modeld/SConscript @@ -69,10 +69,6 @@ if arch == "larch64" or GetOption('pc_thneed'): lenv.Command(fn + ".thneed", [fn + ".onnx"] + tinygrad_files, cmd) - fn_dm = File("models/dmonitoring_model").abspath - cmd = f"cd {Dir('#').abspath}/tinygrad_repo && " + ' '.join(tinygrad_opts) + f" python3 openpilot/compile2.py {fn_dm}.onnx {fn_dm}.thneed" - lenv.Command(fn_dm + ".thneed", [fn_dm + ".onnx"] + tinygrad_files, cmd) - thneed_lib = env.SharedLibrary('thneed', thneed_src, LIBS=[gpucommon, common, 'OpenCL', 'dl']) thneedmodel_lib = env.Library('thneedmodel', ['runners/thneedmodel.cc']) lenvCython.Program('runners/thneedmodel_pyx.so', 'runners/thneedmodel_pyx.pyx', LIBS=envCython["LIBS"]+[thneedmodel_lib, thneed_lib, gpucommon, common, 'dl', 'OpenCL']) diff --git a/selfdrive/modeld/dmonitoringmodeld b/selfdrive/modeld/dmonitoringmodeld deleted file mode 100755 index 80157e1751..0000000000 --- a/selfdrive/modeld/dmonitoringmodeld +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -cd "$DIR/../../" - -if [ -f "$DIR/libthneed.so" ]; then - export LD_PRELOAD="$DIR/libthneed.so" -fi - -exec "$DIR/dmonitoringmodeld.py" "$@" diff --git a/selfdrive/modeld/dmonitoringmodeld.py b/selfdrive/modeld/dmonitoringmodeld.py index d648a4511a..a388bf089c 100755 --- a/selfdrive/modeld/dmonitoringmodeld.py +++ b/selfdrive/modeld/dmonitoringmodeld.py @@ -6,7 +6,6 @@ import time import ctypes import numpy as np from pathlib import Path -from setproctitle import setproctitle from cereal import messaging from cereal.messaging import PubMaster, SubMaster @@ -15,18 +14,16 @@ from openpilot.common.swaglog import cloudlog from openpilot.common.params import Params from openpilot.common.realtime import set_realtime_priority from openpilot.selfdrive.modeld.runners import ModelRunner, Runtime -from openpilot.selfdrive.modeld.models.commonmodel_pyx import sigmoid, CLContext +from openpilot.selfdrive.modeld.models.commonmodel_pyx import sigmoid CALIB_LEN = 3 +REG_SCALE = 0.25 MODEL_WIDTH = 1440 MODEL_HEIGHT = 960 -FEATURE_LEN = 512 -OUTPUT_SIZE = 84 + FEATURE_LEN - -PROCESS_NAME = "selfdrive.modeld.dmonitoringmodeld" +OUTPUT_SIZE = 84 SEND_RAW_PRED = os.getenv('SEND_RAW_PRED') MODEL_PATHS = { - ModelRunner.THNEED: Path(__file__).parent / 'models/dmonitoring_model.thneed', + ModelRunner.SNPE: Path(__file__).parent / 'models/dmonitoring_model_q.dlc', ModelRunner.ONNX: Path(__file__).parent / 'models/dmonitoring_model.onnx'} class DriverStateResult(ctypes.Structure): @@ -52,22 +49,21 @@ class DMonitoringModelResult(ctypes.Structure): ("driver_state_lhd", DriverStateResult), ("driver_state_rhd", DriverStateResult), ("poor_vision_prob", ctypes.c_float), - ("wheel_on_right_prob", ctypes.c_float), - ("features", ctypes.c_float*FEATURE_LEN)] + ("wheel_on_right_prob", ctypes.c_float)] class ModelState: inputs: dict[str, np.ndarray] output: np.ndarray model: ModelRunner - def __init__(self, cl_ctx): + def __init__(self): assert ctypes.sizeof(DMonitoringModelResult) == OUTPUT_SIZE * ctypes.sizeof(ctypes.c_float) self.output = np.zeros(OUTPUT_SIZE, dtype=np.float32) self.inputs = { 'input_img': np.zeros(MODEL_HEIGHT * MODEL_WIDTH, dtype=np.uint8), 'calib': np.zeros(CALIB_LEN, dtype=np.float32)} - self.model = ModelRunner(MODEL_PATHS, self.output, Runtime.GPU, False, cl_ctx) + self.model = ModelRunner(MODEL_PATHS, self.output, Runtime.DSP, True, None) self.model.addInput("input_img", None) self.model.addInput("calib", self.inputs['calib']) @@ -80,17 +76,17 @@ class ModelState: input_data = self.inputs['input_img'].reshape(MODEL_HEIGHT, MODEL_WIDTH) input_data[:] = buf_data[v_offset:v_offset+MODEL_HEIGHT, h_offset:h_offset+MODEL_WIDTH] - self.model.setInputBuffer("input_img", self.inputs['input_img'].view(np.float32)) t1 = time.perf_counter() + self.model.setInputBuffer("input_img", self.inputs['input_img'].view(np.float32)) self.model.execute() t2 = time.perf_counter() return self.output, t2 - t1 def fill_driver_state(msg, ds_result: DriverStateResult): - msg.faceOrientation = list(ds_result.face_orientation) + msg.faceOrientation = [x * REG_SCALE for x in ds_result.face_orientation] msg.faceOrientationStd = [math.exp(x) for x in ds_result.face_orientation_std] - msg.facePosition = list(ds_result.face_position[:2]) + msg.facePosition = [x * REG_SCALE for x in ds_result.face_position[:2]] msg.facePositionStd = [math.exp(x) for x in ds_result.face_position_std[:2]] msg.faceProb = sigmoid(ds_result.face_prob) msg.leftEyeProb = sigmoid(ds_result.left_eye_prob) @@ -119,16 +115,14 @@ def get_driverstate_packet(model_output: np.ndarray, frame_id: int, location_ts: def main(): gc.disable() - setproctitle(PROCESS_NAME) set_realtime_priority(1) - cl_context = CLContext() - model = ModelState(cl_context) + model = ModelState() cloudlog.warning("models loaded, dmonitoringmodeld starting") Params().put_bool("DmModelInitialized", True) cloudlog.warning("connecting to driver stream") - vipc_client = VisionIpcClient("camerad", VisionStreamType.VISION_STREAM_DRIVER, True, cl_context) + vipc_client = VisionIpcClient("camerad", VisionStreamType.VISION_STREAM_DRIVER, True) while not vipc_client.connect(False): time.sleep(0.1) assert vipc_client.is_connected() diff --git a/selfdrive/modeld/models/dmonitoring_model.current b/selfdrive/modeld/models/dmonitoring_model.current index 121871ef2b..f935ab06b3 100644 --- a/selfdrive/modeld/models/dmonitoring_model.current +++ b/selfdrive/modeld/models/dmonitoring_model.current @@ -1,2 +1,2 @@ -fa69be01-b430-4504-9d72-7dcb058eb6dd -d9fb22d1c4fa3ca3d201dbc8edf1d0f0918e53e6 +5ec97a39-0095-4cea-adfa-6d72b1966cc1 +26cac7a9757a27c783a365403040a1bd27ccdaea \ No newline at end of file diff --git a/selfdrive/modeld/models/dmonitoring_model.onnx b/selfdrive/modeld/models/dmonitoring_model.onnx index dcc727510e..dd3a6aba2e 100644 --- a/selfdrive/modeld/models/dmonitoring_model.onnx +++ b/selfdrive/modeld/models/dmonitoring_model.onnx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:50efe6451a3fb3fa04b6bb0e846544533329bd46ecefe9e657e91214dee2aaeb -size 7196502 +oid sha256:3dd3982940d823c4fbb0429b733a0b78b0688d7d67aa76ff7b754a3e2f3d8683 +size 16132780 diff --git a/selfdrive/modeld/models/dmonitoring_model_q.dlc b/selfdrive/modeld/models/dmonitoring_model_q.dlc new file mode 100644 index 0000000000..fc285954d4 --- /dev/null +++ b/selfdrive/modeld/models/dmonitoring_model_q.dlc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c26f13816b143f5bb29ac2980f8557bd5687a75729e4d895313fb9a5a1f0f46 +size 4488449 diff --git a/selfdrive/modeld/runners/onnxmodel.py b/selfdrive/modeld/runners/onnxmodel.py index 2145035a9f..69b44a5a97 100644 --- a/selfdrive/modeld/runners/onnxmodel.py +++ b/selfdrive/modeld/runners/onnxmodel.py @@ -14,12 +14,8 @@ def attributeproto_fp16_to_fp32(attr): attr.data_type = 1 attr.raw_data = float32_list.astype(np.float32).tobytes() -def convert_fp16_to_fp32(onnx_path_or_bytes): - if isinstance(onnx_path_or_bytes, bytes): - model = onnx.load_from_string(onnx_path_or_bytes) - elif isinstance(onnx_path_or_bytes, str): - model = onnx.load(onnx_path_or_bytes) - +def convert_fp16_to_fp32(path): + model = onnx.load(path) for i in model.graph.initializer: if i.data_type == 10: attributeproto_fp16_to_fp32(i) @@ -27,8 +23,6 @@ def convert_fp16_to_fp32(onnx_path_or_bytes): if i.type.tensor_type.elem_type == 10: i.type.tensor_type.elem_type = 1 for i in model.graph.node: - if i.op_type == 'Cast' and i.attribute[0].i == 10: - i.attribute[0].i = 1 for a in i.attribute: if hasattr(a, 't'): if a.t.data_type == 10: @@ -67,6 +61,7 @@ class ONNXModel(RunModel): def __init__(self, path, output, runtime, use_tf8, cl_context): self.inputs = {} self.output = output + self.use_tf8 = use_tf8 self.session = create_ort_session(path, fp16_to_fp32=True) self.input_names = [x.name for x in self.session.get_inputs()] @@ -90,7 +85,7 @@ class ONNXModel(RunModel): return None def execute(self): - inputs = {k: v.view(self.input_dtypes[k]) for k,v in self.inputs.items()} + inputs = {k: (v.view(np.uint8) / 255. if self.use_tf8 and k == 'input_img' else v) for k,v in self.inputs.items()} inputs = {k: v.reshape(self.input_shapes[k]).astype(self.input_dtypes[k]) for k,v in inputs.items()} outputs = self.session.run(None, inputs) assert len(outputs) == 1, "Only single model outputs are supported" diff --git a/selfdrive/monitoring/helpers.py b/selfdrive/monitoring/helpers.py index 08cc049ab0..2a7979221a 100644 --- a/selfdrive/monitoring/helpers.py +++ b/selfdrive/monitoring/helpers.py @@ -33,8 +33,8 @@ class DRIVER_MONITOR_SETTINGS: self._SG_THRESHOLD = 0.9 self._BLINK_THRESHOLD = 0.865 - self._EE_THRESH11 = 0.4 - self._EE_THRESH12 = 15.0 + self._EE_THRESH11 = 0.25 + self._EE_THRESH12 = 7.5 self._EE_MAX_OFFSET1 = 0.06 self._EE_MIN_OFFSET1 = 0.025 self._EE_THRESH21 = 0.01 diff --git a/selfdrive/test/process_replay/model_replay_ref_commit b/selfdrive/test/process_replay/model_replay_ref_commit index 2c757dde11..66971ebde1 100644 --- a/selfdrive/test/process_replay/model_replay_ref_commit +++ b/selfdrive/test/process_replay/model_replay_ref_commit @@ -1 +1 @@ -91d1089681f427a3916b42984d5df04eb94a0b90 +32fe8cf4a0daa8d10a689c9ae2e51a879151c87c diff --git a/selfdrive/test/test_onroad.py b/selfdrive/test/test_onroad.py index 964848e3d3..5bc6bff1c2 100644 --- a/selfdrive/test/test_onroad.py +++ b/selfdrive/test/test_onroad.py @@ -32,7 +32,6 @@ CPU usage budget * total CPU usage of openpilot (sum(PROCS.values()) should not exceed MAX_TOTAL_CPU """ - MAX_TOTAL_CPU = 265. # total for all 8 cores PROCS = { # Baseline CPU usage by process @@ -313,7 +312,7 @@ class TestOnroad: assert max(mems) - min(mems) <= 3.0 def test_gpu_usage(self): - assert self.gpu_procs == {"weston", "ui", "camerad", "selfdrive.modeld.modeld", "selfdrive.modeld.dmonitoringmodeld"} + assert self.gpu_procs == {"weston", "ui", "camerad", "selfdrive.modeld.modeld"} def test_camera_processing_time(self): result = "\n" diff --git a/system/hardware/tici/tests/test_power_draw.py b/system/hardware/tici/tests/test_power_draw.py index 3b9d65b85a..eb3a515b76 100644 --- a/system/hardware/tici/tests/test_power_draw.py +++ b/system/hardware/tici/tests/test_power_draw.py @@ -34,7 +34,7 @@ class Proc: PROCS = [ Proc(['camerad'], 2.1, msgs=['roadCameraState', 'wideRoadCameraState', 'driverCameraState']), Proc(['modeld'], 1.12, atol=0.2, msgs=['modelV2']), - Proc(['dmonitoringmodeld'], 0.5, msgs=['driverStateV2']), + Proc(['dmonitoringmodeld'], 0.4, msgs=['driverStateV2']), Proc(['encoderd'], 0.23, msgs=[]), ] diff --git a/system/manager/process_config.py b/system/manager/process_config.py index 98a875bf97..538c55813b 100644 --- a/system/manager/process_config.py +++ b/system/manager/process_config.py @@ -63,7 +63,7 @@ procs = [ PythonProcess("micd", "system.micd", iscar), PythonProcess("timed", "system.timed", always_run, enabled=not PC), - NativeProcess("dmonitoringmodeld", "selfdrive/modeld", ["./dmonitoringmodeld"], driverview, enabled=(not PC or WEBCAM)), + PythonProcess("dmonitoringmodeld", "selfdrive.modeld.dmonitoringmodeld", driverview, enabled=(not PC or WEBCAM)), NativeProcess("encoderd", "system/loggerd", ["./encoderd"], only_onroad), NativeProcess("stream_encoderd", "system/loggerd", ["./encoderd", "--stream"], notcar), NativeProcess("loggerd", "system/loggerd", ["./loggerd"], logging), diff --git a/tinygrad_repo b/tinygrad_repo index 3e15fa0dae..f51aa0fc7c 160000 --- a/tinygrad_repo +++ b/tinygrad_repo @@ -1 +1 @@ -Subproject commit 3e15fa0daefae75e2ddef98f82be5b5d37820631 +Subproject commit f51aa0fc7cdbac710e640172db280cfb747d2718