wider FOV DM on comma three (#22286)

* c3 is straightforward

* c2 transform

* c3 preview

* alpha model

* Revert "alpha model": wrong quantization data

This reverts commit b50e573973.

* yuv looks good

* a55b

* const int

* block top deadzone

* new box alpha curve

* update k

* fix debug

* rhd use same offset

* box shift for rhd

* 7e3e

* set new thresh

* update ref

* explain

* remove e2e

* same metric thresh

* process isocc

* model

* cancel extra policy

* update numbers

* new model

* update cereal

* up cereal

Co-authored-by: Comma Device <device@comma.ai>

cereal

@@ -1 +1 @@
-Subproject commit 5168470661703a64f99bbd9743f2236ebc2c24ab
+Subproject commit d054cbc70423036a09a9449f7b7b19c058921578

models/dmonitoring_model.current

@@ -1,2 +1,2 @@
-e3926f53-2a15-4a85-8143-dd120f19c050
+4e19be90-bd5b-485d-b79a-2462f7f1b49e
-9208a645dc8f6b251d6fd5899b592243f71d815b
+08f7ec37b78228cd1cb750b6ddb9c6ba1769e911

models/dmonitoring_model.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dea2cce554f1f78919090f7ea93917ae6a319a8ef65f860b3a27026877ff55e7
+oid sha256:895ee32e2a1c77496e015270db475eef65034b25331f2859bac0ccf702f64298
-size 3902922
+size 3294407

models/dmonitoring_model_q.dlc

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d7e8593dd14966195b3fee08be5b53da41b5136f90a2ead8feaf09c3ae898e1
+oid sha256:6e4ac870984d11cd8e86cda4a63e3321fde837bacf4a055a27b7c8ba34facfe2
-size 1068176
+size 916079

selfdrive/modeld/models/dmonitoring.cc

@@ -57,18 +57,21 @@ DMonitoringResult dmonitoring_eval_frame(DMonitoringModelState* s, void* stream_
   if (Hardware::TICI()) {
     const int full_width_tici = 1928;
     const int full_height_tici = 1208;
-    const int adapt_width_tici = 668;
+    const int adapt_width_tici = 954;
+    const int x_offset_tici = -72;
+    const int y_offset_tici = -144;
     const int cropped_height = adapt_width_tici / 1.33;
-    crop_rect = {full_width_tici / 2 - adapt_width_tici / 2,
+    crop_rect = {full_width_tici / 2 - adapt_width_tici / 2 + x_offset_tici,
-                 full_height_tici / 2 - cropped_height / 2 - 196,
+                 full_height_tici / 2 - cropped_height / 2 + y_offset_tici,
                  cropped_height / 2,
                  cropped_height};
     if (!s->is_rhd) {
-      crop_rect.x += adapt_width_tici - crop_rect.w + 32;
+      crop_rect.x += adapt_width_tici - crop_rect.w;
     }
 
   } else {
-    crop_rect = {0, 0, height / 2, height};
+    const int adapt_width = 372;
+    crop_rect = {0, 0, adapt_width, height};
     if (!s->is_rhd) {
       crop_rect.x += width - crop_rect.w;
     }
@@ -95,7 +98,8 @@ DMonitoringResult dmonitoring_eval_frame(DMonitoringModelState* s, void* stream_
   auto [resized_buf, resized_u, resized_v] = get_yuv_buf(s->resized_buf, resized_width, resized_height);
   uint8_t *resized_y = resized_buf;
   libyuv::FilterMode mode = libyuv::FilterModeEnum::kFilterBilinear;
-  libyuv::I420Scale(cropped_y, crop_rect.w,
+  if (Hardware::TICI()) {
+    libyuv::I420Scale(cropped_y, crop_rect.w,
                     cropped_u, crop_rect.w / 2,
                     cropped_v, crop_rect.w / 2,
                     crop_rect.w, crop_rect.h,
@@ -104,6 +108,21 @@ DMonitoringResult dmonitoring_eval_frame(DMonitoringModelState* s, void* stream_
                     resized_v, resized_width / 2,
                     resized_width, resized_height,
                     mode);
+  } else {
+    const int source_height = 0.7*resized_height;
+    const int extra_height = (resized_height - source_height) / 2;
+    const int extra_width = (resized_width - source_height / 2) / 2;
+    const int source_width = source_height / 2 + extra_width;
+    libyuv::I420Scale(cropped_y, crop_rect.w,
+                    cropped_u, crop_rect.w / 2,
+                    cropped_v, crop_rect.w / 2,
+                    crop_rect.w, crop_rect.h,
+                    resized_y + extra_height * resized_width, resized_width,
+                    resized_u + extra_height / 2 * resized_width / 2, resized_width / 2,
+                    resized_v + extra_height / 2 * resized_width / 2, resized_width / 2,
+                    source_width, source_height,
+                    mode);
+  }
 
   int yuv_buf_len = (MODEL_WIDTH/2) * (MODEL_HEIGHT/2) * 6; // Y|u|v -> y|y|y|y|u|v
   float *net_input_buf = get_buffer(s->net_input_buf, yuv_buf_len);
@@ -128,7 +147,7 @@ DMonitoringResult dmonitoring_eval_frame(DMonitoringModelState* s, void* stream_
 
   //printf("preprocess completed. %d \n", yuv_buf_len);
   //FILE *dump_yuv_file = fopen("/tmp/rawdump.yuv", "wb");
-  //fwrite(raw_buf, height*width*3/2, sizeof(uint8_t), dump_yuv_file);
+  //fwrite(resized_buf, yuv_buf_len, sizeof(uint8_t), dump_yuv_file);
   //fclose(dump_yuv_file);
 
   // *** testing ***
@@ -162,6 +181,7 @@ DMonitoringResult dmonitoring_eval_frame(DMonitoringModelState* s, void* stream_
   ret.partial_face = s->output[35];
   ret.distracted_pose = s->output[36];
   ret.distracted_eyes = s->output[37];
+  ret.occluded_prob = s->output[38];
   ret.dsp_execution_time = (t2 - t1) / 1000.;
   return ret;
 }
@@ -188,6 +208,7 @@ void dmonitoring_publish(PubMaster &pm, uint32_t frame_id, const DMonitoringResu
   framed.setPartialFace(res.partial_face);
   framed.setDistractedPose(res.distracted_pose);
   framed.setDistractedEyes(res.distracted_eyes);
+  framed.setOccludedProb(res.occluded_prob);
   if (send_raw_pred) {
     framed.setRawPredictions(raw_pred.asBytes());
   }

selfdrive/modeld/models/dmonitoring.h

@@ -7,7 +7,7 @@
 #include "selfdrive/modeld/models/commonmodel.h"
 #include "selfdrive/modeld/runners/run.h"
 
-#define OUTPUT_SIZE 38
+#define OUTPUT_SIZE 39
 
 typedef struct DMonitoringResult {
   float face_orientation[3];
@@ -24,6 +24,7 @@ typedef struct DMonitoringResult {
   float partial_face;
   float distracted_pose;
   float distracted_eyes;
+  float occluded_prob;
   float dsp_execution_time;
 } DMonitoringResult;
 

selfdrive/monitoring/driver_monitor.py

@@ -26,20 +26,18 @@ class DRIVER_MONITOR_SETTINGS():
     self._DISTRACTED_PROMPT_TIME_TILL_TERMINAL = 6.
 
     self._FACE_THRESHOLD = 0.5
-    self._PARTIAL_FACE_THRESHOLD = 0.765 if TICI else 0.455
+    self._PARTIAL_FACE_THRESHOLD = 0.765 if TICI else 0.43
-    self._EYE_THRESHOLD = 0.25 if TICI else 0.57
+    self._EYE_THRESHOLD = 0.61 if TICI else 0.55
-    self._SG_THRESHOLD = 0.83
+    self._SG_THRESHOLD = 0.89 if TICI else 0.86
-    self._BLINK_THRESHOLD = 0.62 if TICI else 0.68
+    self._BLINK_THRESHOLD = 0.82 if TICI else 0.588
-    self._BLINK_THRESHOLD_SLACK = 0.82 if TICI else 0.88
+    self._BLINK_THRESHOLD_SLACK = 0.9 if TICI else 0.77
-    self._BLINK_THRESHOLD_STRICT = 0.62 if TICI else 0.68
+    self._BLINK_THRESHOLD_STRICT = self._BLINK_THRESHOLD
-    self._PITCH_WEIGHT = 1.175 if TICI else 1.35  # pitch matters a lot more
+    self._PITCH_WEIGHT = 1.35  # pitch matters a lot more
-    self._POSESTD_THRESHOLD = 0.2 if TICI else 0.175
+    self._POSESTD_THRESHOLD = 0.38 if TICI else 0.3
-    self._E2E_POSE_THRESHOLD = 0.95 if TICI else 0.9
+
-    self._E2E_EYES_THRESHOLD = 0.75
+    self._METRIC_THRESHOLD = 0.48
-
+    self._METRIC_THRESHOLD_SLACK = 0.66
-    self._METRIC_THRESHOLD = 0.55 if TICI else 0.48
+    self._METRIC_THRESHOLD_STRICT = self._METRIC_THRESHOLD
-    self._METRIC_THRESHOLD_SLACK = 0.75 if TICI else 0.66
-    self._METRIC_THRESHOLD_STRICT = 0.55 if TICI else 0.48
     self._PITCH_NATURAL_OFFSET = 0.02  # people don't seem to look straight when they drive relaxed, rather a bit up
     self._YAW_NATURAL_OFFSET = 0.08  # people don't seem to look straight when they drive relaxed, rather a bit to the right (center of car)
 
@@ -207,11 +205,8 @@ class DriverStatus():
     self.blink.left_blink = driver_state.leftBlinkProb * (driver_state.leftEyeProb > self.settings._EYE_THRESHOLD) * (driver_state.sunglassesProb < self.settings._SG_THRESHOLD)
     self.blink.right_blink = driver_state.rightBlinkProb * (driver_state.rightEyeProb > self.settings._EYE_THRESHOLD) * (driver_state.sunglassesProb < self.settings._SG_THRESHOLD)
 
-    distracted_normal = self._is_driver_distracted(self.pose, self.blink) > 0 and \
+    self.driver_distracted = self._is_driver_distracted(self.pose, self.blink) > 0 and \
                                    driver_state.faceProb > self.settings._FACE_THRESHOLD and self.pose.low_std
-    distracted_E2E = (driver_state.distractedPose > self.settings._E2E_POSE_THRESHOLD or driver_state.distractedEyes > self.settings._E2E_EYES_THRESHOLD) and \
-                              (self.face_detected and not self.face_partial)
-    self.driver_distracted = distracted_normal or distracted_E2E
     self.driver_distraction_filter.update(self.driver_distracted)
 
     # update offseter

selfdrive/test/process_replay/ref_commit

@@ -1 +1 @@
-b77c895cd7a07b90e7f9ca58252f090a437efc43
+455c2e296f86a9091be35537a58899b07af02506

selfdrive/ui/qt/offroad/driverview.cc

@@ -66,22 +66,27 @@ void DriverViewScene::paintEvent(QPaintEvent* event) {
   const QRect& blackout_rect = Hardware::TICI() ? rect() : rect2;
   p.fillRect(blackout_rect.adjusted(0, 0, valid_rect.left() - blackout_rect.right(), 0), bg);
   p.fillRect(blackout_rect.adjusted(valid_rect.right() - blackout_rect.left(), 0, 0, 0), bg);
+  if (Hardware::TICI()) {
+    p.fillRect(blackout_rect.adjusted(valid_rect.left()-blackout_rect.left()+1, 0, valid_rect.right()-blackout_rect.right()-1, -valid_rect.height()*7/10), bg); // top dz
+  }
 
   // face bounding box
   cereal::DriverState::Reader driver_state = sm["driverState"].getDriverState();
-  bool face_detected = driver_state.getFaceProb() > 0.4;
+  bool face_detected = driver_state.getFaceProb() > 0.5;
   if (face_detected) {
     auto fxy_list = driver_state.getFacePosition();
+    auto std_list = driver_state.getFaceOrientationStd();
     float face_x = fxy_list[0];
     float face_y = fxy_list[1];
+    float face_std = std::max(std_list[0], std_list[1]);
 
-    float alpha = 0.2;
+    float alpha = 0.7;
-    float x = std::abs(face_x), y = std::abs(face_y);
+    if (face_std > 0.08) {
-    if (x <= 0.35 && y <= 0.4) {
+      alpha = std::max(0.7 - (face_std-0.08)*7, 0.0);
-      alpha = 0.8 - std::max(x, y) * 0.6 / 0.375;
     }
     const int box_size = 0.6 * rect2.height() / 2;
-    int fbox_x = valid_rect.center().x() + (is_rhd ? face_x : -face_x) * valid_rect.width();
+    const float rhd_offset = 0.05; // lhd is shifted, so rhd is not mirrored
+    int fbox_x = valid_rect.center().x() + (is_rhd ? (face_x + rhd_offset) : -face_x) * valid_rect.width();
     int fbox_y = valid_rect.center().y() + face_y * valid_rect.height();
     p.setPen(QPen(QColor(255, 255, 255, alpha * 255), 10));
     p.drawRoundedRect(fbox_x - box_size / 2, fbox_y - box_size / 2, box_size, box_size, 35.0, 35.0);

selfdrive/ui/qt/widgets/cameraview.cc

@@ -51,9 +51,9 @@ mat4 get_driver_view_transform() {
     // from dmonitoring.cc
     const int full_width_tici = 1928;
     const int full_height_tici = 1208;
-    const int adapt_width_tici = 668;
+    const int adapt_width_tici = 954;
-    const int crop_x_offset = 32;
+    const int crop_x_offset = -72;
-    const int crop_y_offset = -196;
+    const int crop_y_offset = -144;
     const float yscale = full_height_tici * driver_view_ratio / adapt_width_tici;
     const float xscale = yscale*(1080)/(2160)*full_width_tici/full_height_tici;
     transform = (mat4){{