camerad: use the YUV buffer to create the jpeg thumbnail (#21936)

* yuv420_to_jpeg

continue

* add comments

* cleanup

* return ky::array

selfdrive/camerad/SConscript

@@ -1,6 +1,6 @@
 Import('env', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc', 'USE_WEBCAM')
 
-libs = ['m', 'pthread', common, 'jpeg', 'OpenCL', cereal, messaging, 'zmq', 'capnp', 'kj', visionipc, gpucommon]
+libs = ['m', 'pthread', common, 'jpeg', 'OpenCL', 'yuv', cereal, messaging, 'zmq', 'capnp', 'kj', visionipc, gpucommon]
 
 if arch == "aarch64":
   libs += ['gsl', 'CB', 'adreno_utils', 'EGL', 'GLESv3', 'cutils', 'ui']

selfdrive/camerad/cameras/camera_common.cc

@@ -216,66 +216,84 @@ kj::Array<uint8_t> get_frame_image(const CameraBuf *b) {
   return kj::mv(frame_image);
 }
 
-static void publish_thumbnail(PubMaster *pm, const CameraBuf *b) {
+static kj::Array<capnp::byte> yuv420_to_jpeg(const CameraBuf *b, int thumbnail_width, int thumbnail_height) {
-  uint8_t* thumbnail_buffer = NULL;
+  std::unique_ptr<uint8[]> buf(new uint8_t[(thumbnail_width * thumbnail_height * 3) / 2]);
-  unsigned long thumbnail_len = 0;
+  uint8_t *y_plane = buf.get();
-
+  uint8_t *u_plane = y_plane + thumbnail_width * thumbnail_height;
-  unsigned char *row = (unsigned char *)malloc(b->rgb_width/4*3);
+  uint8_t *v_plane = u_plane + (thumbnail_width * thumbnail_height) / 4;
+  {
+    int result = libyuv::I420Scale(
+        b->cur_yuv_buf->y, b->rgb_width, b->cur_yuv_buf->u, b->rgb_width / 2, b->cur_yuv_buf->v, b->rgb_width / 2,
+        b->rgb_width, b->rgb_height,
+        y_plane, thumbnail_width, u_plane, thumbnail_width / 2, v_plane, thumbnail_width / 2,
+        thumbnail_width, thumbnail_height, libyuv::kFilterNone);
+    if (result != 0) {
+      LOGE("Generate YUV thumbnail failed.");
+      return {};
+    }
+  }
 
   struct jpeg_compress_struct cinfo;
   struct jpeg_error_mgr jerr;
-
   cinfo.err = jpeg_std_error(&jerr);
   jpeg_create_compress(&cinfo);
+
+  uint8_t *thumbnail_buffer = nullptr;
+  size_t thumbnail_len = 0;
   jpeg_mem_dest(&cinfo, &thumbnail_buffer, &thumbnail_len);
 
-  cinfo.image_width = b->rgb_width / 4;
+  cinfo.image_width = thumbnail_width;
-  cinfo.image_height = b->rgb_height / 4;
+  cinfo.image_height = thumbnail_height;
   cinfo.input_components = 3;
-  cinfo.in_color_space = JCS_RGB;
 
   jpeg_set_defaults(&cinfo);
-#ifndef __APPLE__
+  jpeg_set_colorspace(&cinfo, JCS_YCbCr);
-  jpeg_set_quality(&cinfo, 50, true);
+  // configure sampling factors for yuv420.
-  jpeg_start_compress(&cinfo, true);
+  cinfo.comp_info[0].h_samp_factor = 2;  // Y
-#else
+  cinfo.comp_info[0].v_samp_factor = 2;
-  jpeg_set_quality(&cinfo, 50, static_cast<boolean>(true) );
+  cinfo.comp_info[1].h_samp_factor = 1;  // U
-  jpeg_start_compress(&cinfo, static_cast<boolean>(true) );
+  cinfo.comp_info[1].v_samp_factor = 1;
-#endif
+  cinfo.comp_info[2].h_samp_factor = 1;  // V
-
+  cinfo.comp_info[2].v_samp_factor = 1;
-  JSAMPROW row_pointer[1];
+  cinfo.raw_data_in = TRUE;
-  const uint8_t *bgr_ptr = (const uint8_t *)b->cur_rgb_buf->addr;
+
-  for (int ii = 0; ii < b->rgb_height/4; ii+=1) {
+  jpeg_set_quality(&cinfo, 50, TRUE);
-    for (int j = 0; j < b->rgb_width*3; j+=12) {
+  jpeg_start_compress(&cinfo, TRUE);
-      for (int k = 0; k < 3; k++) {
+
-        uint16_t dat = 0;
+  JSAMPROW y[16], u[8], v[8];
-        int i = ii * 4;
+  JSAMPARRAY planes[3]{y, u, v};
-        dat += bgr_ptr[b->rgb_stride*i + j + k];
+
-        dat += bgr_ptr[b->rgb_stride*i + j+3 + k];
+  for (int line = 0; line < cinfo.image_height; line += 16) {
-        dat += bgr_ptr[b->rgb_stride*(i+1) + j + k];
+    for (int i = 0; i < 16; ++i) {
-        dat += bgr_ptr[b->rgb_stride*(i+1) + j+3 + k];
+      y[i] = y_plane + (line + i) * cinfo.image_width;
-        dat += bgr_ptr[b->rgb_stride*(i+2) + j + k];
+      if (i % 2 == 0) {
-        dat += bgr_ptr[b->rgb_stride*(i+2) + j+3 + k];
+        int offset = (cinfo.image_width / 2) * ((i + line) / 2);
-        dat += bgr_ptr[b->rgb_stride*(i+3) + j + k];
+        u[i / 2] = u_plane + offset;
-        dat += bgr_ptr[b->rgb_stride*(i+3) + j+3 + k];
+        v[i / 2] = v_plane + offset;
-        row[(j/4) + (2-k)] = dat/8;
       }
     }
-    row_pointer[0] = row;
+    jpeg_write_raw_data(&cinfo, planes, 16);
-    jpeg_write_scanlines(&cinfo, row_pointer, 1);
   }
+
   jpeg_finish_compress(&cinfo);
   jpeg_destroy_compress(&cinfo);
-  free(row);
+
+  kj::Array<capnp::byte> dat = kj::heapArray<capnp::byte>(thumbnail_buffer, thumbnail_len);
+  free(thumbnail_buffer);
+  return dat;
+}
+
+static void publish_thumbnail(PubMaster *pm, const CameraBuf *b) {
+  auto thumbnail = yuv420_to_jpeg(b, b->rgb_width / 4, b->rgb_height / 4);
+  if (thumbnail.size() == 0) return;
 
   MessageBuilder msg;
   auto thumbnaild = msg.initEvent().initThumbnail();
   thumbnaild.setFrameId(b->cur_frame_data.frame_id);
   thumbnaild.setTimestampEof(b->cur_frame_data.timestamp_eof);
-  thumbnaild.setThumbnail(kj::arrayPtr((const uint8_t*)thumbnail_buffer, thumbnail_len));
+  thumbnaild.setThumbnail(thumbnail);
 
   pm->send("thumbnail", msg);
-  free(thumbnail_buffer);
 }
 
 float set_exposure_target(const CameraBuf *b, int x_start, int x_end, int x_skip, int y_start, int y_end, int y_skip) {