Revert "exec DM model with gpu" (#33604)

Revert "exec DM model with gpu (#33397)"

This reverts commit f79aca8e1e.

.gitattributes

@@ -2,6 +2,7 @@
 
 # to move existing files into LFS:
 # git add --renormalize .
+*.dlc filter=lfs diff=lfs merge=lfs -text
 *.onnx filter=lfs diff=lfs merge=lfs -text
 *.svg filter=lfs diff=lfs merge=lfs -text
 *.png filter=lfs diff=lfs merge=lfs -text

selfdrive/modeld/SConscript

@@ -69,10 +69,6 @@ if arch == "larch64" or GetOption('pc_thneed'):
 
   lenv.Command(fn + ".thneed", [fn + ".onnx"] + tinygrad_files, cmd)
 
-  fn_dm = File("models/dmonitoring_model").abspath
-  cmd = f"cd {Dir('#').abspath}/tinygrad_repo && " + ' '.join(tinygrad_opts) + f" python3 openpilot/compile2.py {fn_dm}.onnx {fn_dm}.thneed"
-  lenv.Command(fn_dm + ".thneed", [fn_dm + ".onnx"] + tinygrad_files, cmd)
-
   thneed_lib = env.SharedLibrary('thneed', thneed_src, LIBS=[gpucommon, common, 'OpenCL', 'dl'])
   thneedmodel_lib = env.Library('thneedmodel', ['runners/thneedmodel.cc'])
   lenvCython.Program('runners/thneedmodel_pyx.so', 'runners/thneedmodel_pyx.pyx', LIBS=envCython["LIBS"]+[thneedmodel_lib, thneed_lib, gpucommon, common, 'dl', 'OpenCL'])

selfdrive/modeld/dmonitoringmodeld

@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
-cd "$DIR/../../"
-
-if [ -f "$DIR/libthneed.so" ]; then
-  export LD_PRELOAD="$DIR/libthneed.so"
-fi
-
-exec "$DIR/dmonitoringmodeld.py" "$@"

selfdrive/modeld/dmonitoringmodeld.py

@@ -6,7 +6,6 @@ import time
 import ctypes
 import numpy as np
 from pathlib import Path
-from setproctitle import setproctitle
 
 from cereal import messaging
 from cereal.messaging import PubMaster, SubMaster
@@ -15,18 +14,16 @@ from openpilot.common.swaglog import cloudlog
 from openpilot.common.params import Params
 from openpilot.common.realtime import set_realtime_priority
 from openpilot.selfdrive.modeld.runners import ModelRunner, Runtime
-from openpilot.selfdrive.modeld.models.commonmodel_pyx import sigmoid, CLContext
+from openpilot.selfdrive.modeld.models.commonmodel_pyx import sigmoid
 
 CALIB_LEN = 3
+REG_SCALE = 0.25
 MODEL_WIDTH = 1440
 MODEL_HEIGHT = 960
-FEATURE_LEN = 512
+OUTPUT_SIZE = 84
-OUTPUT_SIZE = 84 + FEATURE_LEN
-
-PROCESS_NAME = "selfdrive.modeld.dmonitoringmodeld"
 SEND_RAW_PRED = os.getenv('SEND_RAW_PRED')
 MODEL_PATHS = {
-  ModelRunner.THNEED: Path(__file__).parent / 'models/dmonitoring_model.thneed',
+  ModelRunner.SNPE: Path(__file__).parent / 'models/dmonitoring_model_q.dlc',
   ModelRunner.ONNX: Path(__file__).parent / 'models/dmonitoring_model.onnx'}
 
 class DriverStateResult(ctypes.Structure):
@@ -52,22 +49,21 @@ class DMonitoringModelResult(ctypes.Structure):
     ("driver_state_lhd", DriverStateResult),
     ("driver_state_rhd", DriverStateResult),
     ("poor_vision_prob", ctypes.c_float),
-    ("wheel_on_right_prob", ctypes.c_float),
+    ("wheel_on_right_prob", ctypes.c_float)]
-    ("features", ctypes.c_float*FEATURE_LEN)]
 
 class ModelState:
   inputs: dict[str, np.ndarray]
   output: np.ndarray
   model: ModelRunner
 
-  def __init__(self, cl_ctx):
+  def __init__(self):
     assert ctypes.sizeof(DMonitoringModelResult) == OUTPUT_SIZE * ctypes.sizeof(ctypes.c_float)
     self.output = np.zeros(OUTPUT_SIZE, dtype=np.float32)
     self.inputs = {
       'input_img': np.zeros(MODEL_HEIGHT * MODEL_WIDTH, dtype=np.uint8),
       'calib': np.zeros(CALIB_LEN, dtype=np.float32)}
 
-    self.model = ModelRunner(MODEL_PATHS, self.output, Runtime.GPU, False, cl_ctx)
+    self.model = ModelRunner(MODEL_PATHS, self.output, Runtime.DSP, True, None)
     self.model.addInput("input_img", None)
     self.model.addInput("calib", self.inputs['calib'])
 
@@ -80,17 +76,17 @@ class ModelState:
     input_data = self.inputs['input_img'].reshape(MODEL_HEIGHT, MODEL_WIDTH)
     input_data[:] = buf_data[v_offset:v_offset+MODEL_HEIGHT, h_offset:h_offset+MODEL_WIDTH]
 
-    self.model.setInputBuffer("input_img", self.inputs['input_img'].view(np.float32))
     t1 = time.perf_counter()
+    self.model.setInputBuffer("input_img", self.inputs['input_img'].view(np.float32))
     self.model.execute()
     t2 = time.perf_counter()
     return self.output, t2 - t1
 
 
 def fill_driver_state(msg, ds_result: DriverStateResult):
-  msg.faceOrientation = list(ds_result.face_orientation)
+  msg.faceOrientation = [x * REG_SCALE for x in ds_result.face_orientation]
   msg.faceOrientationStd = [math.exp(x) for x in ds_result.face_orientation_std]
-  msg.facePosition = list(ds_result.face_position[:2])
+  msg.facePosition = [x * REG_SCALE for x in ds_result.face_position[:2]]
   msg.facePositionStd = [math.exp(x) for x in ds_result.face_position_std[:2]]
   msg.faceProb = sigmoid(ds_result.face_prob)
   msg.leftEyeProb = sigmoid(ds_result.left_eye_prob)
@@ -119,16 +115,14 @@ def get_driverstate_packet(model_output: np.ndarray, frame_id: int, location_ts:
 
 def main():
   gc.disable()
-  setproctitle(PROCESS_NAME)
   set_realtime_priority(1)
 
-  cl_context = CLContext()
+  model = ModelState()
-  model = ModelState(cl_context)
   cloudlog.warning("models loaded, dmonitoringmodeld starting")
   Params().put_bool("DmModelInitialized", True)
 
   cloudlog.warning("connecting to driver stream")
-  vipc_client = VisionIpcClient("camerad", VisionStreamType.VISION_STREAM_DRIVER, True, cl_context)
+  vipc_client = VisionIpcClient("camerad", VisionStreamType.VISION_STREAM_DRIVER, True)
   while not vipc_client.connect(False):
     time.sleep(0.1)
   assert vipc_client.is_connected()

selfdrive/modeld/models/dmonitoring_model.current

@@ -1,2 +1,2 @@
-fa69be01-b430-4504-9d72-7dcb058eb6dd
+5ec97a39-0095-4cea-adfa-6d72b1966cc1
-d9fb22d1c4fa3ca3d201dbc8edf1d0f0918e53e6
+26cac7a9757a27c783a365403040a1bd27ccdaea

selfdrive/modeld/models/dmonitoring_model.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:50efe6451a3fb3fa04b6bb0e846544533329bd46ecefe9e657e91214dee2aaeb
+oid sha256:3dd3982940d823c4fbb0429b733a0b78b0688d7d67aa76ff7b754a3e2f3d8683
-size 7196502
+size 16132780

selfdrive/modeld/models/dmonitoring_model_q.dlc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c26f13816b143f5bb29ac2980f8557bd5687a75729e4d895313fb9a5a1f0f46
+size 4488449

selfdrive/modeld/runners/onnxmodel.py

@@ -14,12 +14,8 @@ def attributeproto_fp16_to_fp32(attr):
   attr.data_type = 1
   attr.raw_data = float32_list.astype(np.float32).tobytes()
 
-def convert_fp16_to_fp32(onnx_path_or_bytes):
+def convert_fp16_to_fp32(path):
-  if isinstance(onnx_path_or_bytes, bytes):
+  model = onnx.load(path)
-    model = onnx.load_from_string(onnx_path_or_bytes)
-  elif isinstance(onnx_path_or_bytes, str):
-    model = onnx.load(onnx_path_or_bytes)
-
   for i in model.graph.initializer:
     if i.data_type == 10:
       attributeproto_fp16_to_fp32(i)
@@ -27,8 +23,6 @@ def convert_fp16_to_fp32(onnx_path_or_bytes):
     if i.type.tensor_type.elem_type == 10:
       i.type.tensor_type.elem_type = 1
   for i in model.graph.node:
-    if i.op_type == 'Cast' and i.attribute[0].i == 10:
-        i.attribute[0].i = 1
     for a in i.attribute:
       if hasattr(a, 't'):
         if a.t.data_type == 10:
@@ -67,6 +61,7 @@ class ONNXModel(RunModel):
   def __init__(self, path, output, runtime, use_tf8, cl_context):
     self.inputs = {}
     self.output = output
+    self.use_tf8 = use_tf8
 
     self.session = create_ort_session(path, fp16_to_fp32=True)
     self.input_names = [x.name for x in self.session.get_inputs()]
@@ -90,7 +85,7 @@ class ONNXModel(RunModel):
     return None
 
   def execute(self):
-    inputs = {k: v.view(self.input_dtypes[k]) for k,v in self.inputs.items()}
+    inputs = {k: (v.view(np.uint8) / 255. if self.use_tf8 and k == 'input_img' else v) for k,v in self.inputs.items()}
     inputs = {k: v.reshape(self.input_shapes[k]).astype(self.input_dtypes[k]) for k,v in inputs.items()}
     outputs = self.session.run(None, inputs)
     assert len(outputs) == 1, "Only single model outputs are supported"

selfdrive/monitoring/helpers.py

@@ -33,8 +33,8 @@ class DRIVER_MONITOR_SETTINGS:
     self._SG_THRESHOLD = 0.9
     self._BLINK_THRESHOLD = 0.865
 
-    self._EE_THRESH11 = 0.4
+    self._EE_THRESH11 = 0.25
-    self._EE_THRESH12 = 15.0
+    self._EE_THRESH12 = 7.5
     self._EE_MAX_OFFSET1 = 0.06
     self._EE_MIN_OFFSET1 = 0.025
     self._EE_THRESH21 = 0.01

selfdrive/test/process_replay/model_replay_ref_commit

@@ -1 +1 @@
-91d1089681f427a3916b42984d5df04eb94a0b90
+32fe8cf4a0daa8d10a689c9ae2e51a879151c87c

selfdrive/test/test_onroad.py

@@ -32,7 +32,6 @@ CPU usage budget
 * total CPU usage of openpilot (sum(PROCS.values())
   should not exceed MAX_TOTAL_CPU
 """
-
 MAX_TOTAL_CPU = 265.  # total for all 8 cores
 PROCS = {
   # Baseline CPU usage by process
@@ -313,7 +312,7 @@ class TestOnroad:
     assert max(mems) - min(mems) <= 3.0
 
   def test_gpu_usage(self):
-    assert self.gpu_procs == {"weston", "ui", "camerad", "selfdrive.modeld.modeld", "selfdrive.modeld.dmonitoringmodeld"}
+    assert self.gpu_procs == {"weston", "ui", "camerad", "selfdrive.modeld.modeld"}
 
   def test_camera_processing_time(self):
     result = "\n"

system/hardware/tici/tests/test_power_draw.py

@@ -34,7 +34,7 @@ class Proc:
 PROCS = [
   Proc(['camerad'], 2.1, msgs=['roadCameraState', 'wideRoadCameraState', 'driverCameraState']),
   Proc(['modeld'], 1.12, atol=0.2, msgs=['modelV2']),
-  Proc(['dmonitoringmodeld'], 0.5, msgs=['driverStateV2']),
+  Proc(['dmonitoringmodeld'], 0.4, msgs=['driverStateV2']),
   Proc(['encoderd'], 0.23, msgs=[]),
 ]
 

system/manager/process_config.py

@@ -63,7 +63,7 @@ procs = [
   PythonProcess("micd", "system.micd", iscar),
   PythonProcess("timed", "system.timed", always_run, enabled=not PC),
 
-  NativeProcess("dmonitoringmodeld", "selfdrive/modeld", ["./dmonitoringmodeld"], driverview, enabled=(not PC or WEBCAM)),
+  PythonProcess("dmonitoringmodeld", "selfdrive.modeld.dmonitoringmodeld", driverview, enabled=(not PC or WEBCAM)),
   NativeProcess("encoderd", "system/loggerd", ["./encoderd"], only_onroad),
   NativeProcess("stream_encoderd", "system/loggerd", ["./encoderd", "--stream"], notcar),
   NativeProcess("loggerd", "system/loggerd", ["./loggerd"], logging),

tinygrad_repo

@@ -1 +1 @@
-Subproject commit 3e15fa0daefae75e2ddef98f82be5b5d37820631
+Subproject commit f51aa0fc7cdbac710e640172db280cfb747d2718