From f79aca8e1ef9160a93b2609146b4074b2343f75c Mon Sep 17 00:00:00 2001 From: ZwX1616 Date: Thu, 19 Sep 2024 17:42:44 -0700 Subject: [PATCH] exec DM model with gpu (#33397) * half old-commit-hash: 9f72eca003d4637ca7fe847ebaf925c694fc2e84 * optimed old-commit-hash: 6e36e2a12e09275ec21d1590012a92b05ca52ff5 * thneed old-commit-hash: 419a06c09c0c767d828bcd1e118bc575898c343a * exec old-commit-hash: 0059c27ec11b076a37f65d604ed135ea6541b1a6 * runner old-commit-hash: 34232ada94450ce541eaef546197fa219810a891 * runs but old-commit-hash: 3db37c00b6a64908293b4de8d8b56e80308cd8f2 * it is 01 old-commit-hash: a160d81eb1a7e77abbef959b44f602610f68f665 * np old-commit-hash: c1caff6ba648cc2c0094c71b2ea074f01c3c2dc8 * module url old-commit-hash: 6f4902c4d384263a53e2c1d14d93b5ff864b6a5f * new old-commit-hash: 779ae79b1bc3df6374fb6663ac8592e107a6e504 * ds fast * is this work * corcention * real timing * no reg * interim gather * 0e4a9c7b * fa69be01, and halve * list * cleanup * slighly faster * setprotlt * expected * replay ref * more powar * reluctantly * bump tg * 8 * less * less * bump tg * better than exp * closer * cc * see diff * commits * was right * to 32 cast * remove dlc file * support both --------- Co-authored-by: Comma Device --- .gitattributes | 1 - selfdrive/modeld/SConscript | 4 +++ selfdrive/modeld/dmonitoringmodeld | 10 +++++++ selfdrive/modeld/dmonitoringmodeld.py | 30 +++++++++++-------- .../modeld/models/dmonitoring_model.current | 4 +-- .../modeld/models/dmonitoring_model.onnx | 4 +-- .../modeld/models/dmonitoring_model_q.dlc | 3 -- selfdrive/modeld/runners/onnxmodel.py | 13 +++++--- selfdrive/monitoring/helpers.py | 4 +-- .../process_replay/model_replay_ref_commit | 2 +- selfdrive/test/test_onroad.py | 3 +- system/hardware/tici/tests/test_power_draw.py | 2 +- system/manager/process_config.py | 2 +- tinygrad_repo | 2 +- 14 files changed, 53 insertions(+), 31 deletions(-) create mode 100755 selfdrive/modeld/dmonitoringmodeld delete mode 100644 selfdrive/modeld/models/dmonitoring_model_q.dlc diff --git a/.gitattributes b/.gitattributes index 8781a7371f..eda2505d0e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,7 +2,6 @@ # to move existing files into LFS: # git add --renormalize . -*.dlc filter=lfs diff=lfs merge=lfs -text *.onnx filter=lfs diff=lfs merge=lfs -text *.svg filter=lfs diff=lfs merge=lfs -text *.png filter=lfs diff=lfs merge=lfs -text diff --git a/selfdrive/modeld/SConscript b/selfdrive/modeld/SConscript index d572915c72..d472998416 100644 --- a/selfdrive/modeld/SConscript +++ b/selfdrive/modeld/SConscript @@ -69,6 +69,10 @@ if arch == "larch64" or GetOption('pc_thneed'): lenv.Command(fn + ".thneed", [fn + ".onnx"] + tinygrad_files, cmd) + fn_dm = File("models/dmonitoring_model").abspath + cmd = f"cd {Dir('#').abspath}/tinygrad_repo && " + ' '.join(tinygrad_opts) + f" python3 openpilot/compile2.py {fn_dm}.onnx {fn_dm}.thneed" + lenv.Command(fn_dm + ".thneed", [fn_dm + ".onnx"] + tinygrad_files, cmd) + thneed_lib = env.SharedLibrary('thneed', thneed_src, LIBS=[gpucommon, common, 'OpenCL', 'dl']) thneedmodel_lib = env.Library('thneedmodel', ['runners/thneedmodel.cc']) lenvCython.Program('runners/thneedmodel_pyx.so', 'runners/thneedmodel_pyx.pyx', LIBS=envCython["LIBS"]+[thneedmodel_lib, thneed_lib, gpucommon, common, 'dl', 'OpenCL']) diff --git a/selfdrive/modeld/dmonitoringmodeld b/selfdrive/modeld/dmonitoringmodeld new file mode 100755 index 0000000000..80157e1751 --- /dev/null +++ b/selfdrive/modeld/dmonitoringmodeld @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +cd "$DIR/../../" + +if [ -f "$DIR/libthneed.so" ]; then + export LD_PRELOAD="$DIR/libthneed.so" +fi + +exec "$DIR/dmonitoringmodeld.py" "$@" diff --git a/selfdrive/modeld/dmonitoringmodeld.py b/selfdrive/modeld/dmonitoringmodeld.py index a388bf089c..d648a4511a 100755 --- a/selfdrive/modeld/dmonitoringmodeld.py +++ b/selfdrive/modeld/dmonitoringmodeld.py @@ -6,6 +6,7 @@ import time import ctypes import numpy as np from pathlib import Path +from setproctitle import setproctitle from cereal import messaging from cereal.messaging import PubMaster, SubMaster @@ -14,16 +15,18 @@ from openpilot.common.swaglog import cloudlog from openpilot.common.params import Params from openpilot.common.realtime import set_realtime_priority from openpilot.selfdrive.modeld.runners import ModelRunner, Runtime -from openpilot.selfdrive.modeld.models.commonmodel_pyx import sigmoid +from openpilot.selfdrive.modeld.models.commonmodel_pyx import sigmoid, CLContext CALIB_LEN = 3 -REG_SCALE = 0.25 MODEL_WIDTH = 1440 MODEL_HEIGHT = 960 -OUTPUT_SIZE = 84 +FEATURE_LEN = 512 +OUTPUT_SIZE = 84 + FEATURE_LEN + +PROCESS_NAME = "selfdrive.modeld.dmonitoringmodeld" SEND_RAW_PRED = os.getenv('SEND_RAW_PRED') MODEL_PATHS = { - ModelRunner.SNPE: Path(__file__).parent / 'models/dmonitoring_model_q.dlc', + ModelRunner.THNEED: Path(__file__).parent / 'models/dmonitoring_model.thneed', ModelRunner.ONNX: Path(__file__).parent / 'models/dmonitoring_model.onnx'} class DriverStateResult(ctypes.Structure): @@ -49,21 +52,22 @@ class DMonitoringModelResult(ctypes.Structure): ("driver_state_lhd", DriverStateResult), ("driver_state_rhd", DriverStateResult), ("poor_vision_prob", ctypes.c_float), - ("wheel_on_right_prob", ctypes.c_float)] + ("wheel_on_right_prob", ctypes.c_float), + ("features", ctypes.c_float*FEATURE_LEN)] class ModelState: inputs: dict[str, np.ndarray] output: np.ndarray model: ModelRunner - def __init__(self): + def __init__(self, cl_ctx): assert ctypes.sizeof(DMonitoringModelResult) == OUTPUT_SIZE * ctypes.sizeof(ctypes.c_float) self.output = np.zeros(OUTPUT_SIZE, dtype=np.float32) self.inputs = { 'input_img': np.zeros(MODEL_HEIGHT * MODEL_WIDTH, dtype=np.uint8), 'calib': np.zeros(CALIB_LEN, dtype=np.float32)} - self.model = ModelRunner(MODEL_PATHS, self.output, Runtime.DSP, True, None) + self.model = ModelRunner(MODEL_PATHS, self.output, Runtime.GPU, False, cl_ctx) self.model.addInput("input_img", None) self.model.addInput("calib", self.inputs['calib']) @@ -76,17 +80,17 @@ class ModelState: input_data = self.inputs['input_img'].reshape(MODEL_HEIGHT, MODEL_WIDTH) input_data[:] = buf_data[v_offset:v_offset+MODEL_HEIGHT, h_offset:h_offset+MODEL_WIDTH] - t1 = time.perf_counter() self.model.setInputBuffer("input_img", self.inputs['input_img'].view(np.float32)) + t1 = time.perf_counter() self.model.execute() t2 = time.perf_counter() return self.output, t2 - t1 def fill_driver_state(msg, ds_result: DriverStateResult): - msg.faceOrientation = [x * REG_SCALE for x in ds_result.face_orientation] + msg.faceOrientation = list(ds_result.face_orientation) msg.faceOrientationStd = [math.exp(x) for x in ds_result.face_orientation_std] - msg.facePosition = [x * REG_SCALE for x in ds_result.face_position[:2]] + msg.facePosition = list(ds_result.face_position[:2]) msg.facePositionStd = [math.exp(x) for x in ds_result.face_position_std[:2]] msg.faceProb = sigmoid(ds_result.face_prob) msg.leftEyeProb = sigmoid(ds_result.left_eye_prob) @@ -115,14 +119,16 @@ def get_driverstate_packet(model_output: np.ndarray, frame_id: int, location_ts: def main(): gc.disable() + setproctitle(PROCESS_NAME) set_realtime_priority(1) - model = ModelState() + cl_context = CLContext() + model = ModelState(cl_context) cloudlog.warning("models loaded, dmonitoringmodeld starting") Params().put_bool("DmModelInitialized", True) cloudlog.warning("connecting to driver stream") - vipc_client = VisionIpcClient("camerad", VisionStreamType.VISION_STREAM_DRIVER, True) + vipc_client = VisionIpcClient("camerad", VisionStreamType.VISION_STREAM_DRIVER, True, cl_context) while not vipc_client.connect(False): time.sleep(0.1) assert vipc_client.is_connected() diff --git a/selfdrive/modeld/models/dmonitoring_model.current b/selfdrive/modeld/models/dmonitoring_model.current index f935ab06b3..121871ef2b 100644 --- a/selfdrive/modeld/models/dmonitoring_model.current +++ b/selfdrive/modeld/models/dmonitoring_model.current @@ -1,2 +1,2 @@ -5ec97a39-0095-4cea-adfa-6d72b1966cc1 -26cac7a9757a27c783a365403040a1bd27ccdaea \ No newline at end of file +fa69be01-b430-4504-9d72-7dcb058eb6dd +d9fb22d1c4fa3ca3d201dbc8edf1d0f0918e53e6 diff --git a/selfdrive/modeld/models/dmonitoring_model.onnx b/selfdrive/modeld/models/dmonitoring_model.onnx index dd3a6aba2e..dcc727510e 100644 --- a/selfdrive/modeld/models/dmonitoring_model.onnx +++ b/selfdrive/modeld/models/dmonitoring_model.onnx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3dd3982940d823c4fbb0429b733a0b78b0688d7d67aa76ff7b754a3e2f3d8683 -size 16132780 +oid sha256:50efe6451a3fb3fa04b6bb0e846544533329bd46ecefe9e657e91214dee2aaeb +size 7196502 diff --git a/selfdrive/modeld/models/dmonitoring_model_q.dlc b/selfdrive/modeld/models/dmonitoring_model_q.dlc deleted file mode 100644 index fc285954d4..0000000000 --- a/selfdrive/modeld/models/dmonitoring_model_q.dlc +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7c26f13816b143f5bb29ac2980f8557bd5687a75729e4d895313fb9a5a1f0f46 -size 4488449 diff --git a/selfdrive/modeld/runners/onnxmodel.py b/selfdrive/modeld/runners/onnxmodel.py index 69b44a5a97..2145035a9f 100644 --- a/selfdrive/modeld/runners/onnxmodel.py +++ b/selfdrive/modeld/runners/onnxmodel.py @@ -14,8 +14,12 @@ def attributeproto_fp16_to_fp32(attr): attr.data_type = 1 attr.raw_data = float32_list.astype(np.float32).tobytes() -def convert_fp16_to_fp32(path): - model = onnx.load(path) +def convert_fp16_to_fp32(onnx_path_or_bytes): + if isinstance(onnx_path_or_bytes, bytes): + model = onnx.load_from_string(onnx_path_or_bytes) + elif isinstance(onnx_path_or_bytes, str): + model = onnx.load(onnx_path_or_bytes) + for i in model.graph.initializer: if i.data_type == 10: attributeproto_fp16_to_fp32(i) @@ -23,6 +27,8 @@ def convert_fp16_to_fp32(path): if i.type.tensor_type.elem_type == 10: i.type.tensor_type.elem_type = 1 for i in model.graph.node: + if i.op_type == 'Cast' and i.attribute[0].i == 10: + i.attribute[0].i = 1 for a in i.attribute: if hasattr(a, 't'): if a.t.data_type == 10: @@ -61,7 +67,6 @@ class ONNXModel(RunModel): def __init__(self, path, output, runtime, use_tf8, cl_context): self.inputs = {} self.output = output - self.use_tf8 = use_tf8 self.session = create_ort_session(path, fp16_to_fp32=True) self.input_names = [x.name for x in self.session.get_inputs()] @@ -85,7 +90,7 @@ class ONNXModel(RunModel): return None def execute(self): - inputs = {k: (v.view(np.uint8) / 255. if self.use_tf8 and k == 'input_img' else v) for k,v in self.inputs.items()} + inputs = {k: v.view(self.input_dtypes[k]) for k,v in self.inputs.items()} inputs = {k: v.reshape(self.input_shapes[k]).astype(self.input_dtypes[k]) for k,v in inputs.items()} outputs = self.session.run(None, inputs) assert len(outputs) == 1, "Only single model outputs are supported" diff --git a/selfdrive/monitoring/helpers.py b/selfdrive/monitoring/helpers.py index 2a7979221a..08cc049ab0 100644 --- a/selfdrive/monitoring/helpers.py +++ b/selfdrive/monitoring/helpers.py @@ -33,8 +33,8 @@ class DRIVER_MONITOR_SETTINGS: self._SG_THRESHOLD = 0.9 self._BLINK_THRESHOLD = 0.865 - self._EE_THRESH11 = 0.25 - self._EE_THRESH12 = 7.5 + self._EE_THRESH11 = 0.4 + self._EE_THRESH12 = 15.0 self._EE_MAX_OFFSET1 = 0.06 self._EE_MIN_OFFSET1 = 0.025 self._EE_THRESH21 = 0.01 diff --git a/selfdrive/test/process_replay/model_replay_ref_commit b/selfdrive/test/process_replay/model_replay_ref_commit index 66971ebde1..2c757dde11 100644 --- a/selfdrive/test/process_replay/model_replay_ref_commit +++ b/selfdrive/test/process_replay/model_replay_ref_commit @@ -1 +1 @@ -32fe8cf4a0daa8d10a689c9ae2e51a879151c87c +91d1089681f427a3916b42984d5df04eb94a0b90 diff --git a/selfdrive/test/test_onroad.py b/selfdrive/test/test_onroad.py index 5bc6bff1c2..964848e3d3 100644 --- a/selfdrive/test/test_onroad.py +++ b/selfdrive/test/test_onroad.py @@ -32,6 +32,7 @@ CPU usage budget * total CPU usage of openpilot (sum(PROCS.values()) should not exceed MAX_TOTAL_CPU """ + MAX_TOTAL_CPU = 265. # total for all 8 cores PROCS = { # Baseline CPU usage by process @@ -312,7 +313,7 @@ class TestOnroad: assert max(mems) - min(mems) <= 3.0 def test_gpu_usage(self): - assert self.gpu_procs == {"weston", "ui", "camerad", "selfdrive.modeld.modeld"} + assert self.gpu_procs == {"weston", "ui", "camerad", "selfdrive.modeld.modeld", "selfdrive.modeld.dmonitoringmodeld"} def test_camera_processing_time(self): result = "\n" diff --git a/system/hardware/tici/tests/test_power_draw.py b/system/hardware/tici/tests/test_power_draw.py index eb3a515b76..3b9d65b85a 100644 --- a/system/hardware/tici/tests/test_power_draw.py +++ b/system/hardware/tici/tests/test_power_draw.py @@ -34,7 +34,7 @@ class Proc: PROCS = [ Proc(['camerad'], 2.1, msgs=['roadCameraState', 'wideRoadCameraState', 'driverCameraState']), Proc(['modeld'], 1.12, atol=0.2, msgs=['modelV2']), - Proc(['dmonitoringmodeld'], 0.4, msgs=['driverStateV2']), + Proc(['dmonitoringmodeld'], 0.5, msgs=['driverStateV2']), Proc(['encoderd'], 0.23, msgs=[]), ] diff --git a/system/manager/process_config.py b/system/manager/process_config.py index 538c55813b..98a875bf97 100644 --- a/system/manager/process_config.py +++ b/system/manager/process_config.py @@ -63,7 +63,7 @@ procs = [ PythonProcess("micd", "system.micd", iscar), PythonProcess("timed", "system.timed", always_run, enabled=not PC), - PythonProcess("dmonitoringmodeld", "selfdrive.modeld.dmonitoringmodeld", driverview, enabled=(not PC or WEBCAM)), + NativeProcess("dmonitoringmodeld", "selfdrive/modeld", ["./dmonitoringmodeld"], driverview, enabled=(not PC or WEBCAM)), NativeProcess("encoderd", "system/loggerd", ["./encoderd"], only_onroad), NativeProcess("stream_encoderd", "system/loggerd", ["./encoderd", "--stream"], notcar), NativeProcess("loggerd", "system/loggerd", ["./loggerd"], logging), diff --git a/tinygrad_repo b/tinygrad_repo index f51aa0fc7c..3e15fa0dae 160000 --- a/tinygrad_repo +++ b/tinygrad_repo @@ -1 +1 @@ -Subproject commit f51aa0fc7cdbac710e640172db280cfb747d2718 +Subproject commit 3e15fa0daefae75e2ddef98f82be5b5d37820631