Set GPU priorities + improved modeld priorities (#2691)

* give gpu threads rt priority

* modeld gets highest prio on core 2

* fix frame stream

* lower prio of cal thread

Co-authored-by: Comma Device <device@comma.ai>

common/realtime.py

@@ -21,9 +21,14 @@ else:
 
 
 class Priority:
-  MIN_REALTIME = 52 # highest android process priority is 51
+  # CORE 2
-  CTRL_LOW = MIN_REALTIME
+  # - modeld = 55
-  CTRL_HIGH = MIN_REALTIME + 1
+  # - camerad = 54
+  CTRL_LOW = 51 # plannerd & radard
+
+  # CORE 3
+  # - boardd = 55
+  CTRL_HIGH = 53
 
 
 def set_realtime_priority(level):

launch_chffrplus.sh

@@ -69,6 +69,11 @@ function two_init {
   done
   echo 2 > /proc/irq/193/smp_affinity_list # GPU
 
+  # give GPU threads RT priority
+  for pid in $(pgrep "kgsl"); do
+    chrt -f -p 52 $pid
+  done
+
   # Check for NEOS update
   if [ $(< /VERSION) != "$REQUIRED_NEOS_VERSION" ]; then
     if [ -f "$DIR/scripts/continue.sh" ]; then

selfdrive/camerad/cameras/camera_common.cc

@@ -53,7 +53,7 @@ void CameraBuf::init(cl_device_id device_id, cl_context context, CameraState *s,
   camera_state = s;
   frame_buf_count = frame_cnt;
   frame_size = ci->frame_height * ci->frame_stride;
- 
+
   camera_bufs = std::make_unique<VisionBuf[]>(frame_buf_count);
   camera_bufs_metadata = std::make_unique<FrameMetadata[]>(frame_buf_count);
   for (int i = 0; i < frame_buf_count; i++) {
@@ -351,10 +351,8 @@ void set_exposure_target(CameraState *c, const uint8_t *pix_ptr, int x_start, in
 extern volatile sig_atomic_t do_exit;
 
 void *processing_thread(MultiCameraState *cameras, const char *tname,
-                        CameraState *cs, int priority, process_thread_cb callback) {
+                          CameraState *cs, process_thread_cb callback) {
   set_thread_name(tname);
-  int err = set_realtime_priority(priority);
-  LOG("%s start! setpriority returns %d", tname, err);
 
   for (int cnt = 0; !do_exit; cnt++) {
     if (!cs->buf.acquire()) continue;
@@ -367,8 +365,8 @@ void *processing_thread(MultiCameraState *cameras, const char *tname,
 }
 
 std::thread start_process_thread(MultiCameraState *cameras, const char *tname,
-                                 CameraState *cs, int priority, process_thread_cb callback) {
+                                 CameraState *cs, process_thread_cb callback) {
-  return std::thread(processing_thread, cameras, tname, cs, priority, callback);
+  return std::thread(processing_thread, cameras, tname, cs, callback);
 }
 
 void common_camera_process_front(SubMaster *sm, PubMaster *pm, CameraState *c, int cnt) {

selfdrive/camerad/cameras/camera_common.h

@@ -138,5 +138,5 @@ void fill_frame_image(cereal::FrameData::Builder &framed, uint8_t *dat, int w, i
 void create_thumbnail(MultiCameraState *s, CameraState *c, uint8_t *bgr_ptr);
 void set_exposure_target(CameraState *c, const uint8_t *pix_ptr, int x_start, int x_end, int x_skip, int y_start, int y_end, int y_skip);
 std::thread start_process_thread(MultiCameraState *cameras, const char *tname,
-                                    CameraState *cs, int priority, process_thread_cb callback);
+                                    CameraState *cs, process_thread_cb callback);
 void common_camera_process_front(SubMaster *sm, PubMaster *pm, CameraState *c, int cnt);

selfdrive/camerad/cameras/camera_frame_stream.cc

@@ -126,7 +126,7 @@ void camera_process_rear(MultiCameraState *s, CameraState *c, int cnt) {
 }
 
 void cameras_run(MultiCameraState *s) {
-  std::thread t = start_process_thread(s, "processing", &s->rear, 51, camera_process_rear);
+  std::thread t = start_process_thread(s, "processing", &s->rear, camera_process_rear);
   set_thread_name("frame_streaming");
   run_frame_stream(s);
   cameras_close(s);

selfdrive/camerad/cameras/camera_qcom.cc

@@ -2147,8 +2147,8 @@ void cameras_run(MultiCameraState *s) {
                        ops_thread, s);
   assert(err == 0);
   std::vector<std::thread> threads;
-  threads.push_back(start_process_thread(s, "processing", &s->rear, 51, camera_process_frame));
+  threads.push_back(start_process_thread(s, "processing", &s->rear, camera_process_frame));
-  threads.push_back(start_process_thread(s, "frontview", &s->front, 51, camera_process_front));
+  threads.push_back(start_process_thread(s, "frontview", &s->front, camera_process_front));
 
   CameraState* cameras[2] = {&s->rear, &s->front};
 

selfdrive/camerad/cameras/camera_qcom2.cc

@@ -1161,9 +1161,9 @@ void cameras_run(MultiCameraState *s) {
                        ae_thread, s);
   assert(err == 0);
   std::vector<std::thread> threads;
-  threads.push_back(start_process_thread(s, "processing", &s->rear, 51, camera_process_frame));
+  threads.push_back(start_process_thread(s, "processing", &s->rear, camera_process_frame));
-  threads.push_back(start_process_thread(s, "frontview", &s->front, 51, camera_process_front));
+  threads.push_back(start_process_thread(s, "frontview", &s->front, camera_process_front));
-  threads.push_back(start_process_thread(s, "wideview", &s->wide, 51, camera_process_frame));
+  threads.push_back(start_process_thread(s, "wideview", &s->wide, camera_process_frame));
 
   // start devices
   LOG("-- Starting devices");

selfdrive/camerad/cameras/camera_webcam.cc

@@ -272,14 +272,14 @@ void camera_process_rear(MultiCameraState *s, CameraState *c, int cnt) {
 
 void cameras_run(MultiCameraState *s) {
   std::vector<std::thread> threads;
-  threads.push_back(start_process_thread(s, "processing", &s->rear, 51, camera_process_rear));
+  threads.push_back(start_process_thread(s, "processing", &s->rear, camera_process_rear));
-  threads.push_back(start_process_thread(s, "frontview", &s->front, 51, camera_process_front));
+  threads.push_back(start_process_thread(s, "frontview", &s->front, camera_process_front));
-  
+
   std::thread t_rear = std::thread(rear_thread, &s->rear);
   set_thread_name("webcam_thread");
   front_thread(&s->front);
   t_rear.join();
   cameras_close(s);
-  
+
   for (auto &t : threads) t.join();
 }

selfdrive/camerad/main.cc

@@ -304,14 +304,14 @@ void* visionserver_thread(void* arg) {
 void party(cl_device_id device_id, cl_context context) {
   VisionState state = {};
   VisionState *s = &state;
-  
+
   cameras_init(&s->cameras, device_id, context);
   cameras_open(&s->cameras);
 
   std::thread server_thread(visionserver_thread, s);
-  
+
   // priority for cameras
-  int err = set_realtime_priority(51);
+  int err = set_realtime_priority(53);
   LOG("setpriority returns %d", err);
 
   cameras_run(&s->cameras);
@@ -324,7 +324,7 @@ void party(cl_device_id device_id, cl_context context) {
 #endif
 
 int main(int argc, char *argv[]) {
-  set_realtime_priority(51);
+  set_realtime_priority(53);
 #if defined(QCOM)
   set_core_affinity(2);
 #elif defined(QCOM2)

selfdrive/manager.py

@@ -233,6 +233,7 @@ car_started_processes = [
   'calibrationd',
   'paramsd',
   'camerad',
+  'modeld',
   'proclogd',
   'locationd',
   'clocksd',
@@ -263,9 +264,6 @@ if ANDROID:
     'rtshield',
   ]
 
-# starting dmonitoringmodeld when modeld is initializing can sometimes \
-# result in a weird snpe state where dmon constantly uses more cpu than normal.
-car_started_processes += ['modeld']
 
 def register_managed_process(name, desc, car_started=False):
   global managed_processes, car_started_processes, persistent_processes

selfdrive/modeld/modeld.cc

@@ -24,8 +24,10 @@ pthread_mutex_t transform_lock;
 
 void* live_thread(void *arg) {
   set_thread_name("live");
+  set_realtime_priority(50);
 
   SubMaster sm({"liveCalibration"});
+
   /*
      import numpy as np
      from common.transformations.model import medmodel_frame_from_road_frame
@@ -60,7 +62,7 @@ void* live_thread(void *arg) {
   }}, db_s);
 
   while (!do_exit) {
-    if (sm.update(10) > 0){
+    if (sm.update(100) > 0){
 
       auto extrinsic_matrix = sm["liveCalibration"].getLiveCalibration().getExtrinsicMatrix();
       Eigen::Matrix<float, 3, 4> extrinsic_matrix_eigen;
@@ -91,7 +93,7 @@ void* live_thread(void *arg) {
 
 int main(int argc, char **argv) {
   int err;
-  set_realtime_priority(51);
+  set_realtime_priority(54);
 
 #ifdef QCOM
   set_core_affinity(2);