From 8574e79fecff6dba6c68486b627c73499d4b3d03 Mon Sep 17 00:00:00 2001 From: ZwX1616 Date: Wed, 13 Oct 2021 19:00:46 -0500 Subject: [PATCH] wider FOV DM on comma three (#22286) * c3 is straightforward * c2 transform * c3 preview * alpha model * Revert "alpha model": wrong quantization data This reverts commit b50e5739735b30d960aca22bcf3cbe23b0121e13. * yuv looks good * a55b * const int * block top deadzone * new box alpha curve * update k * fix debug * rhd use same offset * box shift for rhd * 7e3e * set new thresh * update ref * explain * remove e2e * same metric thresh * process isocc * model * cancel extra policy * update numbers * new model * update cereal * up cereal Co-authored-by: Comma Device --- cereal | 2 +- models/dmonitoring_model.current | 4 +-- models/dmonitoring_model.onnx | 4 +-- models/dmonitoring_model_q.dlc | 4 +-- selfdrive/modeld/models/dmonitoring.cc | 35 +++++++++++++++++++----- selfdrive/modeld/models/dmonitoring.h | 3 +- selfdrive/monitoring/driver_monitor.py | 31 +++++++++------------ selfdrive/test/process_replay/ref_commit | 2 +- selfdrive/ui/qt/offroad/driverview.cc | 17 ++++++++---- selfdrive/ui/qt/widgets/cameraview.cc | 6 ++-- 10 files changed, 65 insertions(+), 43 deletions(-) diff --git a/cereal b/cereal index 5168470661..d054cbc704 160000 --- a/cereal +++ b/cereal @@ -1 +1 @@ -Subproject commit 5168470661703a64f99bbd9743f2236ebc2c24ab +Subproject commit d054cbc70423036a09a9449f7b7b19c058921578 diff --git a/models/dmonitoring_model.current b/models/dmonitoring_model.current index c9a70ac64e..b9e0c24751 100644 --- a/models/dmonitoring_model.current +++ b/models/dmonitoring_model.current @@ -1,2 +1,2 @@ -e3926f53-2a15-4a85-8143-dd120f19c050 -9208a645dc8f6b251d6fd5899b592243f71d815b \ No newline at end of file +4e19be90-bd5b-485d-b79a-2462f7f1b49e +08f7ec37b78228cd1cb750b6ddb9c6ba1769e911 \ No newline at end of file diff --git a/models/dmonitoring_model.onnx b/models/dmonitoring_model.onnx index 3e0e8abe56..9fe743334d 100644 --- a/models/dmonitoring_model.onnx +++ b/models/dmonitoring_model.onnx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:dea2cce554f1f78919090f7ea93917ae6a319a8ef65f860b3a27026877ff55e7 -size 3902922 +oid sha256:895ee32e2a1c77496e015270db475eef65034b25331f2859bac0ccf702f64298 +size 3294407 diff --git a/models/dmonitoring_model_q.dlc b/models/dmonitoring_model_q.dlc index cd28e6691c..1e34e0de3f 100644 --- a/models/dmonitoring_model_q.dlc +++ b/models/dmonitoring_model_q.dlc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5d7e8593dd14966195b3fee08be5b53da41b5136f90a2ead8feaf09c3ae898e1 -size 1068176 +oid sha256:6e4ac870984d11cd8e86cda4a63e3321fde837bacf4a055a27b7c8ba34facfe2 +size 916079 diff --git a/selfdrive/modeld/models/dmonitoring.cc b/selfdrive/modeld/models/dmonitoring.cc index 2711db45ef..719a9d8a07 100644 --- a/selfdrive/modeld/models/dmonitoring.cc +++ b/selfdrive/modeld/models/dmonitoring.cc @@ -57,18 +57,21 @@ DMonitoringResult dmonitoring_eval_frame(DMonitoringModelState* s, void* stream_ if (Hardware::TICI()) { const int full_width_tici = 1928; const int full_height_tici = 1208; - const int adapt_width_tici = 668; + const int adapt_width_tici = 954; + const int x_offset_tici = -72; + const int y_offset_tici = -144; const int cropped_height = adapt_width_tici / 1.33; - crop_rect = {full_width_tici / 2 - adapt_width_tici / 2, - full_height_tici / 2 - cropped_height / 2 - 196, + crop_rect = {full_width_tici / 2 - adapt_width_tici / 2 + x_offset_tici, + full_height_tici / 2 - cropped_height / 2 + y_offset_tici, cropped_height / 2, cropped_height}; if (!s->is_rhd) { - crop_rect.x += adapt_width_tici - crop_rect.w + 32; + crop_rect.x += adapt_width_tici - crop_rect.w; } } else { - crop_rect = {0, 0, height / 2, height}; + const int adapt_width = 372; + crop_rect = {0, 0, adapt_width, height}; if (!s->is_rhd) { crop_rect.x += width - crop_rect.w; } @@ -95,7 +98,8 @@ DMonitoringResult dmonitoring_eval_frame(DMonitoringModelState* s, void* stream_ auto [resized_buf, resized_u, resized_v] = get_yuv_buf(s->resized_buf, resized_width, resized_height); uint8_t *resized_y = resized_buf; libyuv::FilterMode mode = libyuv::FilterModeEnum::kFilterBilinear; - libyuv::I420Scale(cropped_y, crop_rect.w, + if (Hardware::TICI()) { + libyuv::I420Scale(cropped_y, crop_rect.w, cropped_u, crop_rect.w / 2, cropped_v, crop_rect.w / 2, crop_rect.w, crop_rect.h, @@ -104,6 +108,21 @@ DMonitoringResult dmonitoring_eval_frame(DMonitoringModelState* s, void* stream_ resized_v, resized_width / 2, resized_width, resized_height, mode); + } else { + const int source_height = 0.7*resized_height; + const int extra_height = (resized_height - source_height) / 2; + const int extra_width = (resized_width - source_height / 2) / 2; + const int source_width = source_height / 2 + extra_width; + libyuv::I420Scale(cropped_y, crop_rect.w, + cropped_u, crop_rect.w / 2, + cropped_v, crop_rect.w / 2, + crop_rect.w, crop_rect.h, + resized_y + extra_height * resized_width, resized_width, + resized_u + extra_height / 2 * resized_width / 2, resized_width / 2, + resized_v + extra_height / 2 * resized_width / 2, resized_width / 2, + source_width, source_height, + mode); + } int yuv_buf_len = (MODEL_WIDTH/2) * (MODEL_HEIGHT/2) * 6; // Y|u|v -> y|y|y|y|u|v float *net_input_buf = get_buffer(s->net_input_buf, yuv_buf_len); @@ -128,7 +147,7 @@ DMonitoringResult dmonitoring_eval_frame(DMonitoringModelState* s, void* stream_ //printf("preprocess completed. %d \n", yuv_buf_len); //FILE *dump_yuv_file = fopen("/tmp/rawdump.yuv", "wb"); - //fwrite(raw_buf, height*width*3/2, sizeof(uint8_t), dump_yuv_file); + //fwrite(resized_buf, yuv_buf_len, sizeof(uint8_t), dump_yuv_file); //fclose(dump_yuv_file); // *** testing *** @@ -162,6 +181,7 @@ DMonitoringResult dmonitoring_eval_frame(DMonitoringModelState* s, void* stream_ ret.partial_face = s->output[35]; ret.distracted_pose = s->output[36]; ret.distracted_eyes = s->output[37]; + ret.occluded_prob = s->output[38]; ret.dsp_execution_time = (t2 - t1) / 1000.; return ret; } @@ -188,6 +208,7 @@ void dmonitoring_publish(PubMaster &pm, uint32_t frame_id, const DMonitoringResu framed.setPartialFace(res.partial_face); framed.setDistractedPose(res.distracted_pose); framed.setDistractedEyes(res.distracted_eyes); + framed.setOccludedProb(res.occluded_prob); if (send_raw_pred) { framed.setRawPredictions(raw_pred.asBytes()); } diff --git a/selfdrive/modeld/models/dmonitoring.h b/selfdrive/modeld/models/dmonitoring.h index 3015374732..9b3ee631f0 100644 --- a/selfdrive/modeld/models/dmonitoring.h +++ b/selfdrive/modeld/models/dmonitoring.h @@ -7,7 +7,7 @@ #include "selfdrive/modeld/models/commonmodel.h" #include "selfdrive/modeld/runners/run.h" -#define OUTPUT_SIZE 38 +#define OUTPUT_SIZE 39 typedef struct DMonitoringResult { float face_orientation[3]; @@ -24,6 +24,7 @@ typedef struct DMonitoringResult { float partial_face; float distracted_pose; float distracted_eyes; + float occluded_prob; float dsp_execution_time; } DMonitoringResult; diff --git a/selfdrive/monitoring/driver_monitor.py b/selfdrive/monitoring/driver_monitor.py index a09e3dfcf8..e4018d66f6 100644 --- a/selfdrive/monitoring/driver_monitor.py +++ b/selfdrive/monitoring/driver_monitor.py @@ -26,20 +26,18 @@ class DRIVER_MONITOR_SETTINGS(): self._DISTRACTED_PROMPT_TIME_TILL_TERMINAL = 6. self._FACE_THRESHOLD = 0.5 - self._PARTIAL_FACE_THRESHOLD = 0.765 if TICI else 0.455 - self._EYE_THRESHOLD = 0.25 if TICI else 0.57 - self._SG_THRESHOLD = 0.83 - self._BLINK_THRESHOLD = 0.62 if TICI else 0.68 - self._BLINK_THRESHOLD_SLACK = 0.82 if TICI else 0.88 - self._BLINK_THRESHOLD_STRICT = 0.62 if TICI else 0.68 - self._PITCH_WEIGHT = 1.175 if TICI else 1.35 # pitch matters a lot more - self._POSESTD_THRESHOLD = 0.2 if TICI else 0.175 - self._E2E_POSE_THRESHOLD = 0.95 if TICI else 0.9 - self._E2E_EYES_THRESHOLD = 0.75 - - self._METRIC_THRESHOLD = 0.55 if TICI else 0.48 - self._METRIC_THRESHOLD_SLACK = 0.75 if TICI else 0.66 - self._METRIC_THRESHOLD_STRICT = 0.55 if TICI else 0.48 + self._PARTIAL_FACE_THRESHOLD = 0.765 if TICI else 0.43 + self._EYE_THRESHOLD = 0.61 if TICI else 0.55 + self._SG_THRESHOLD = 0.89 if TICI else 0.86 + self._BLINK_THRESHOLD = 0.82 if TICI else 0.588 + self._BLINK_THRESHOLD_SLACK = 0.9 if TICI else 0.77 + self._BLINK_THRESHOLD_STRICT = self._BLINK_THRESHOLD + self._PITCH_WEIGHT = 1.35 # pitch matters a lot more + self._POSESTD_THRESHOLD = 0.38 if TICI else 0.3 + + self._METRIC_THRESHOLD = 0.48 + self._METRIC_THRESHOLD_SLACK = 0.66 + self._METRIC_THRESHOLD_STRICT = self._METRIC_THRESHOLD self._PITCH_NATURAL_OFFSET = 0.02 # people don't seem to look straight when they drive relaxed, rather a bit up self._YAW_NATURAL_OFFSET = 0.08 # people don't seem to look straight when they drive relaxed, rather a bit to the right (center of car) @@ -207,11 +205,8 @@ class DriverStatus(): self.blink.left_blink = driver_state.leftBlinkProb * (driver_state.leftEyeProb > self.settings._EYE_THRESHOLD) * (driver_state.sunglassesProb < self.settings._SG_THRESHOLD) self.blink.right_blink = driver_state.rightBlinkProb * (driver_state.rightEyeProb > self.settings._EYE_THRESHOLD) * (driver_state.sunglassesProb < self.settings._SG_THRESHOLD) - distracted_normal = self._is_driver_distracted(self.pose, self.blink) > 0 and \ + self.driver_distracted = self._is_driver_distracted(self.pose, self.blink) > 0 and \ driver_state.faceProb > self.settings._FACE_THRESHOLD and self.pose.low_std - distracted_E2E = (driver_state.distractedPose > self.settings._E2E_POSE_THRESHOLD or driver_state.distractedEyes > self.settings._E2E_EYES_THRESHOLD) and \ - (self.face_detected and not self.face_partial) - self.driver_distracted = distracted_normal or distracted_E2E self.driver_distraction_filter.update(self.driver_distracted) # update offseter diff --git a/selfdrive/test/process_replay/ref_commit b/selfdrive/test/process_replay/ref_commit index 508d5b7c82..a5fa157f93 100644 --- a/selfdrive/test/process_replay/ref_commit +++ b/selfdrive/test/process_replay/ref_commit @@ -1 +1 @@ -b77c895cd7a07b90e7f9ca58252f090a437efc43 \ No newline at end of file +455c2e296f86a9091be35537a58899b07af02506 \ No newline at end of file diff --git a/selfdrive/ui/qt/offroad/driverview.cc b/selfdrive/ui/qt/offroad/driverview.cc index af9750c230..db2a4d6b40 100644 --- a/selfdrive/ui/qt/offroad/driverview.cc +++ b/selfdrive/ui/qt/offroad/driverview.cc @@ -66,22 +66,27 @@ void DriverViewScene::paintEvent(QPaintEvent* event) { const QRect& blackout_rect = Hardware::TICI() ? rect() : rect2; p.fillRect(blackout_rect.adjusted(0, 0, valid_rect.left() - blackout_rect.right(), 0), bg); p.fillRect(blackout_rect.adjusted(valid_rect.right() - blackout_rect.left(), 0, 0, 0), bg); + if (Hardware::TICI()) { + p.fillRect(blackout_rect.adjusted(valid_rect.left()-blackout_rect.left()+1, 0, valid_rect.right()-blackout_rect.right()-1, -valid_rect.height()*7/10), bg); // top dz + } // face bounding box cereal::DriverState::Reader driver_state = sm["driverState"].getDriverState(); - bool face_detected = driver_state.getFaceProb() > 0.4; + bool face_detected = driver_state.getFaceProb() > 0.5; if (face_detected) { auto fxy_list = driver_state.getFacePosition(); + auto std_list = driver_state.getFaceOrientationStd(); float face_x = fxy_list[0]; float face_y = fxy_list[1]; + float face_std = std::max(std_list[0], std_list[1]); - float alpha = 0.2; - float x = std::abs(face_x), y = std::abs(face_y); - if (x <= 0.35 && y <= 0.4) { - alpha = 0.8 - std::max(x, y) * 0.6 / 0.375; + float alpha = 0.7; + if (face_std > 0.08) { + alpha = std::max(0.7 - (face_std-0.08)*7, 0.0); } const int box_size = 0.6 * rect2.height() / 2; - int fbox_x = valid_rect.center().x() + (is_rhd ? face_x : -face_x) * valid_rect.width(); + const float rhd_offset = 0.05; // lhd is shifted, so rhd is not mirrored + int fbox_x = valid_rect.center().x() + (is_rhd ? (face_x + rhd_offset) : -face_x) * valid_rect.width(); int fbox_y = valid_rect.center().y() + face_y * valid_rect.height(); p.setPen(QPen(QColor(255, 255, 255, alpha * 255), 10)); p.drawRoundedRect(fbox_x - box_size / 2, fbox_y - box_size / 2, box_size, box_size, 35.0, 35.0); diff --git a/selfdrive/ui/qt/widgets/cameraview.cc b/selfdrive/ui/qt/widgets/cameraview.cc index 339e0e7e38..dd3af9e209 100644 --- a/selfdrive/ui/qt/widgets/cameraview.cc +++ b/selfdrive/ui/qt/widgets/cameraview.cc @@ -51,9 +51,9 @@ mat4 get_driver_view_transform() { // from dmonitoring.cc const int full_width_tici = 1928; const int full_height_tici = 1208; - const int adapt_width_tici = 668; - const int crop_x_offset = 32; - const int crop_y_offset = -196; + const int adapt_width_tici = 954; + const int crop_x_offset = -72; + const int crop_y_offset = -144; const float yscale = full_height_tici * driver_view_ratio / adapt_width_tici; const float xscale = yscale*(1080)/(2160)*full_width_tici/full_height_tici; transform = (mat4){{