camerad: setup IFE black level and color correction (#33834)

* black level

* enable cc

* seems to work

* doesnt do anything

* sensorinfo

* this is fine

* cleanup

* disable

* cleanup os

* revert

---------

Co-authored-by: Comma Device <device@comma.ai>
Co-authored-by: ZwX1616 <zwx1616@gmail.com>

system/camerad/SConscript

@@ -2,9 +2,10 @@ Import('env', 'arch', 'messaging', 'common', 'gpucommon', 'visionipc')
 
 libs = ['pthread', common, 'jpeg', 'OpenCL', messaging, visionipc, gpucommon]
 
-camera_obj = env.Object(['cameras/camera_qcom2.cc', 'cameras/camera_common.cc', 'cameras/spectra.cc',
-                         'cameras/cdm.cc', 'sensors/ar0231.cc', 'sensors/ox03c10.cc', 'sensors/os04c10.cc'])
-env.Program('camerad', ['main.cc', camera_obj], LIBS=libs)
+if arch != "Darwin":
+  camera_obj = env.Object(['cameras/camera_qcom2.cc', 'cameras/camera_common.cc', 'cameras/spectra.cc',
+                           'cameras/cdm.cc', 'sensors/ar0231.cc', 'sensors/ox03c10.cc', 'sensors/os04c10.cc'])
+  env.Program('camerad', ['main.cc', camera_obj], LIBS=libs)
 
 if GetOption("extras") and arch == "x86_64":
   env.Program('test/test_ae_gray', ['test/test_ae_gray.cc', camera_obj], LIBS=libs)

system/camerad/cameras/camera_qcom2.cc

@@ -12,7 +12,12 @@
 #include <string>
 #include <vector>
 
+#ifdef QCOM2
 #include "CL/cl_ext_qcom.h"
+#else
+#define CL_PRIORITY_HINT_HIGH_QCOM NULL
+#define CL_CONTEXT_PRIORITY_HINT_QCOM NULL
+#endif
 
 #include "media/cam_sensor_cmn_header.h"
 
@@ -50,7 +55,7 @@ public:
 
   float fl_pix = 0;
 
-  CameraState(SpectraMaster *master, const CameraConfig &config) : camera(master, config, true /*config.camera_num == 2*/) {};
+  CameraState(SpectraMaster *master, const CameraConfig &config) : camera(master, config, true /*config.stream_type == VISION_STREAM_ROAD*/) {};
   ~CameraState();
   void init(VisionIpcServer *v, cl_device_id device_id, cl_context ctx);
   void update_exposure_score(float desired_ev, int exp_t, int exp_g_idx, float exp_gain);
@@ -283,7 +288,7 @@ void camerad_thread() {
 
   // *** per-cam init ***
   std::vector<std::unique_ptr<CameraState>> cams;
-  for (const auto &config : {ROAD_CAMERA_CONFIG, WIDE_ROAD_CAMERA_CONFIG, DRIVER_CAMERA_CONFIG}) {
+  for (const auto &config : {WIDE_ROAD_CAMERA_CONFIG, DRIVER_CAMERA_CONFIG, ROAD_CAMERA_CONFIG}) {
     auto cam = std::make_unique<CameraState>(&m, config);
     cam->init(&v, device_id, ctx);
     cams.emplace_back(std::move(cam));

system/camerad/cameras/ife.h

@@ -1,9 +1,12 @@
 #include "cdm.h"
 
-int build_initial_config(uint8_t *dst) {
+#include "system/camerad/cameras/tici.h"
+#include "system/camerad/sensors/sensor.h"
+
+
+int build_initial_config(uint8_t *dst, const SensorInfo *s) {
   uint8_t *start = dst;
 
-  // constants, some kind of HW quirk?
   dst += write_random(dst, {
     0x2c, 0xffffffff,
     0x30, 0xffffffff,
@@ -173,21 +176,7 @@ int build_initial_config(uint8_t *dst) {
     0x08000066,
   });
 
-  dst += write_cont(dst, 0x760, {
-    0x00800080,
-    0x00000000,
-    0x00000000,
-    0x00000000,
-    0x00800080,
-    0x00000000,
-    0x00000000,
-    0x00000000,
-    0x00800080,
-    0x00000000,
-    0x00000000,
-    0x00000000,
-    0x00000000,
-  });
+  dst += write_cont(dst, 0x760, s->color_correct_matrix);
 
   dst += write_cont(dst, 0x794, {
     0x00000000,
@@ -655,7 +644,7 @@ int build_first_update(uint8_t *dst) {
   return dst - start;
 }
 
-int build_update(uint8_t *dst) {
+int build_update(uint8_t *dst, const CameraConfig &cc, const SensorInfo *s) {
   uint8_t *start = dst;
 
   dst += write_random(dst, {
@@ -708,11 +697,11 @@ int build_update(uint8_t *dst) {
   });
 
   dst += write_cont(dst, 0x40, {
-    0x00000444,
+    0x00000c04,
   });
 
   dst += write_cont(dst, 0x48, {
-    0x00000000,
+    0b10,
   });
 
   dst += write_cont(dst, 0x4c, {
@@ -763,5 +752,14 @@ int build_update(uint8_t *dst) {
     0x00000000,
   });
 
+  // *** extras, not in original dump ***
+
+  // black level scale + offset
+  dst += write_cont(dst, 0x6b0, {
+    (((uint32_t)(1 << s->bits_per_pixel) - 1) << 0xf) | (s->black_level << 0),
+    0x0,
+    0x0,
+  });
+
   return dst - start;
 }

system/camerad/cameras/spectra.cc

@@ -495,11 +495,11 @@ void SpectraCamera::config_ife(int idx, int request_id, bool init) {
     // stream of IFE register writes
     if (!is_raw) {
       if (init) {
-        buf_desc[0].length = build_initial_config((unsigned char*)ife_cmd.ptr + buf_desc[0].offset);
+        buf_desc[0].length = build_initial_config((unsigned char*)ife_cmd.ptr + buf_desc[0].offset, sensor.get());
       } else if (request_id == 1) {
         buf_desc[0].length = build_first_update((unsigned char*)ife_cmd.ptr + buf_desc[0].offset);
       } else {
-        buf_desc[0].length = build_update((unsigned char*)ife_cmd.ptr + buf_desc[0].offset);
+        buf_desc[0].length = build_update((unsigned char*)ife_cmd.ptr + buf_desc[0].offset, cc, sensor.get());
       }
     }
 

system/camerad/sensors/ar0231.cc

@@ -93,6 +93,7 @@ AR0231::AR0231() {
   init_reg_array.assign(std::begin(init_array_ar0231), std::end(init_array_ar0231));
   probe_reg_addr = 0x3000;
   probe_expected_data = 0x354;
+  bits_per_pixel = 12;
   mipi_format = CAM_FORMAT_MIPI_RAW_12;
   frame_data_type = 0x12;  // Changing stats to 0x2C doesn't work, so change pixels to 0x12 instead
   mclk_frequency = 19200000; //Hz
@@ -116,6 +117,13 @@ AR0231::AR0231() {
   min_ev = exposure_time_min * sensor_analog_gains[analog_gain_min_idx];
   max_ev = exposure_time_max * dc_gain_factor * sensor_analog_gains[analog_gain_max_idx];
   target_grey_factor = 1.0;
+
+  black_level = 168;
+  color_correct_matrix = {
+    0x000000af, 0x00000ff9, 0x00000fd8,
+    0x00000fbc, 0x000000bb, 0x00000009,
+    0x00000fb6, 0x00000fe0, 0x000000ea,
+  };
 }
 
 void AR0231::processRegisters(uint8_t *cur_buf, cereal::FrameData::Builder &framed) const {

system/camerad/sensors/os04c10.cc

@@ -37,6 +37,7 @@ OS04C10::OS04C10() {
   init_reg_array.assign(std::begin(init_array_os04c10), std::end(init_array_os04c10));
   probe_reg_addr = 0x300a;
   probe_expected_data = 0x5304;
+  bits_per_pixel = 10;
   mipi_format = CAM_FORMAT_MIPI_RAW_10;
   frame_data_type = 0x2b;
   mclk_frequency = 24000000; // Hz
@@ -60,6 +61,13 @@ OS04C10::OS04C10() {
   min_ev = (exposure_time_min) * sensor_analog_gains[analog_gain_min_idx];
   max_ev = exposure_time_max * dc_gain_factor * sensor_analog_gains[analog_gain_max_idx];
   target_grey_factor = 0.01;
+
+  black_level = 64;
+  color_correct_matrix = {
+    0x000000c2, 0x00000fe0, 0x00000fde,
+    0x00000fa7, 0x000000d9, 0x00001000,
+    0x00000fca, 0x00000fef, 0x000000c7,
+  };
 }
 
 std::vector<i2c_random_wr_payload> OS04C10::getExposureRegisters(int exposure_time, int new_exp_g, bool dc_gain_enabled) const {

system/camerad/sensors/ox03c10.cc

@@ -37,6 +37,7 @@ OX03C10::OX03C10() {
   init_reg_array.assign(std::begin(init_array_ox03c10), std::end(init_array_ox03c10));
   probe_reg_addr = 0x300a;
   probe_expected_data = 0x5803;
+  bits_per_pixel = 12;
   mipi_format = CAM_FORMAT_MIPI_RAW_12;
   frame_data_type = 0x2c; // one is 0x2a, two are 0x2b
   mclk_frequency = 24000000; //Hz
@@ -60,6 +61,13 @@ OX03C10::OX03C10() {
   min_ev = (exposure_time_min + VS_TIME_MIN_OX03C10) * sensor_analog_gains[analog_gain_min_idx];
   max_ev = exposure_time_max * dc_gain_factor * sensor_analog_gains[analog_gain_max_idx];
   target_grey_factor = 0.01;
+
+  black_level = 64;
+  color_correct_matrix = {
+    0x000000b6, 0x00000ff1, 0x00000fda,
+    0x00000fcc, 0x000000b9, 0x00000ffb,
+    0x00000fc2, 0x00000ff6, 0x000000c9,
+  };
 }
 
 std::vector<i2c_random_wr_payload> OX03C10::getExposureRegisters(int exposure_time, int new_exp_g, bool dc_gain_enabled) const {

system/camerad/sensors/sensor.h

@@ -60,10 +60,15 @@ public:
   std::vector<i2c_random_wr_payload> start_reg_array;
   std::vector<i2c_random_wr_payload> init_reg_array;
 
+  uint32_t bits_per_pixel;
   uint32_t bayer_pattern = CAM_ISP_PATTERN_BAYER_GRGRGR;
   uint32_t mipi_format;
   uint32_t mclk_frequency;
   uint32_t frame_data_type;
+
+  // ISP image processing params
+  uint32_t black_level;
+  std::vector<uint32_t> color_correct_matrix;
 };
 
 class AR0231 : public SensorInfo {