move thumbnail generation from camerad to encoderd (#34554)

Co-authored-by: Cameron Clough <cameronjclough@gmail.com>

cereal/services.py

@@ -53,7 +53,7 @@ _services: dict[str, tuple] = {
   "livePose": (True, 20., 4),
   "liveParameters": (True, 20., 5),
   "cameraOdometry": (True, 20., 10),
-  "thumbnail": (True, 0.2, 1),
+  "thumbnail": (True, 1 / 60., 1),
   "onroadEvents": (True, 1., 1),
   "carParams": (True, 0.02, 1),
   "roadCameraState": (True, 20., 20),

system/camerad/SConscript

@@ -1,6 +1,6 @@
 Import('env', 'arch', 'messaging', 'common', 'gpucommon', 'visionipc')
 
-libs = ['pthread', common, 'jpeg', 'OpenCL', messaging, visionipc, gpucommon]
+libs = ['pthread', common, 'OpenCL', messaging, visionipc, gpucommon]
 
 if arch != "Darwin":
   camera_obj = env.Object(['cameras/camera_qcom2.cc', 'cameras/camera_common.cc', 'cameras/spectra.cc',

system/camerad/cameras/camera_common.cc

@@ -3,8 +3,6 @@
 #include <cassert>
 #include <string>
 
-#include <jpeglib.h>
-
 #include "common/clutil.h"
 #include "common/swaglog.h"
 
@@ -150,95 +148,6 @@ kj::Array<uint8_t> get_raw_frame_image(const CameraBuf *b) {
   return kj::mv(frame_image);
 }
 
-static kj::Array<capnp::byte> yuv420_to_jpeg(const CameraBuf *b, int thumbnail_width, int thumbnail_height) {
-  int downscale = b->cur_yuv_buf->width / thumbnail_width;
-  assert(downscale * thumbnail_height == b->cur_yuv_buf->height);
-  int in_stride = b->cur_yuv_buf->stride;
-
-  // make the buffer big enough. jpeg_write_raw_data requires 16-pixels aligned height to be used.
-  std::unique_ptr<uint8_t[]> buf(new uint8_t[(thumbnail_width * ((thumbnail_height + 15) & ~15) * 3) / 2]);
-  uint8_t *y_plane = buf.get();
-  uint8_t *u_plane = y_plane + thumbnail_width * thumbnail_height;
-  uint8_t *v_plane = u_plane + (thumbnail_width * thumbnail_height) / 4;
-  {
-    // subsampled conversion from nv12 to yuv
-    for (int hy = 0; hy < thumbnail_height/2; hy++) {
-      for (int hx = 0; hx < thumbnail_width/2; hx++) {
-        int ix = hx * downscale + (downscale-1)/2;
-        int iy = hy * downscale + (downscale-1)/2;
-        y_plane[(hy*2 + 0)*thumbnail_width + (hx*2 + 0)] = b->cur_yuv_buf->y[(iy*2 + 0) * in_stride + ix*2 + 0];
-        y_plane[(hy*2 + 0)*thumbnail_width + (hx*2 + 1)] = b->cur_yuv_buf->y[(iy*2 + 0) * in_stride + ix*2 + 1];
-        y_plane[(hy*2 + 1)*thumbnail_width + (hx*2 + 0)] = b->cur_yuv_buf->y[(iy*2 + 1) * in_stride + ix*2 + 0];
-        y_plane[(hy*2 + 1)*thumbnail_width + (hx*2 + 1)] = b->cur_yuv_buf->y[(iy*2 + 1) * in_stride + ix*2 + 1];
-        u_plane[hy*thumbnail_width/2 + hx] = b->cur_yuv_buf->uv[iy*in_stride + ix*2 + 0];
-        v_plane[hy*thumbnail_width/2 + hx] = b->cur_yuv_buf->uv[iy*in_stride + ix*2 + 1];
-      }
-    }
-  }
-
-  struct jpeg_compress_struct cinfo;
-  struct jpeg_error_mgr jerr;
-  cinfo.err = jpeg_std_error(&jerr);
-  jpeg_create_compress(&cinfo);
-
-  uint8_t *thumbnail_buffer = nullptr;
-  size_t thumbnail_len = 0;
-  jpeg_mem_dest(&cinfo, &thumbnail_buffer, &thumbnail_len);
-
-  cinfo.image_width = thumbnail_width;
-  cinfo.image_height = thumbnail_height;
-  cinfo.input_components = 3;
-
-  jpeg_set_defaults(&cinfo);
-  jpeg_set_colorspace(&cinfo, JCS_YCbCr);
-  // configure sampling factors for yuv420.
-  cinfo.comp_info[0].h_samp_factor = 2;  // Y
-  cinfo.comp_info[0].v_samp_factor = 2;
-  cinfo.comp_info[1].h_samp_factor = 1;  // U
-  cinfo.comp_info[1].v_samp_factor = 1;
-  cinfo.comp_info[2].h_samp_factor = 1;  // V
-  cinfo.comp_info[2].v_samp_factor = 1;
-  cinfo.raw_data_in = TRUE;
-
-  jpeg_set_quality(&cinfo, 50, TRUE);
-  jpeg_start_compress(&cinfo, TRUE);
-
-  JSAMPROW y[16], u[8], v[8];
-  JSAMPARRAY planes[3]{y, u, v};
-
-  for (int line = 0; line < cinfo.image_height; line += 16) {
-    for (int i = 0; i < 16; ++i) {
-      y[i] = y_plane + (line + i) * cinfo.image_width;
-      if (i % 2 == 0) {
-        int offset = (cinfo.image_width / 2) * ((i + line) / 2);
-        u[i / 2] = u_plane + offset;
-        v[i / 2] = v_plane + offset;
-      }
-    }
-    jpeg_write_raw_data(&cinfo, planes, 16);
-  }
-
-  jpeg_finish_compress(&cinfo);
-  jpeg_destroy_compress(&cinfo);
-
-  kj::Array<capnp::byte> dat = kj::heapArray<capnp::byte>(thumbnail_buffer, thumbnail_len);
-  free(thumbnail_buffer);
-  return dat;
-}
-
-void publish_thumbnail(PubMaster *pm, const CameraBuf *b) {
-  auto thumbnail = yuv420_to_jpeg(b, b->out_img_width / 4, b->out_img_height / 4);
-  if (thumbnail.size() == 0) return;
-
-  MessageBuilder msg;
-  auto thumbnaild = msg.initEvent().initThumbnail();
-  thumbnaild.setFrameId(b->cur_frame_data.frame_id);
-  thumbnaild.setTimestampEof(b->cur_frame_data.timestamp_eof);
-  thumbnaild.setThumbnail(thumbnail);
-
-  pm->send("thumbnail", msg);
-}
-
 float set_exposure_target(const CameraBuf *b, Rect ae_xywh, int x_skip, int y_skip) {
   int lum_med;
   uint32_t lum_binning[256] = {0};

system/camerad/cameras/camera_common.h

@@ -52,5 +52,4 @@ public:
 void camerad_thread();
 kj::Array<uint8_t> get_raw_frame_image(const CameraBuf *b);
 float set_exposure_target(const CameraBuf *b, Rect ae_xywh, int x_skip, int y_skip);
-void publish_thumbnail(PubMaster *pm, const CameraBuf *b);
 int open_v4l_by_name_and_index(const char name[], int index = 0, int flags = O_RDWR | O_NONBLOCK);

system/camerad/cameras/camera_qcom2.cc

@@ -231,9 +231,7 @@ void CameraState::set_camera_exposure(float grey_frac) {
 void CameraState::run() {
   util::set_thread_name(camera.cc.publish_name);
 
-  std::vector<const char*> pubs = {camera.cc.publish_name};
+  PubMaster pm(std::vector{camera.cc.publish_name});
-  if (camera.cc.stream_type == VISION_STREAM_ROAD) pubs.push_back("thumbnail");
-  PubMaster pm(pubs);
 
   for (uint32_t cnt = 0; !do_exit; ++cnt) {
     // Acquire the buffer; continue if acquisition fails
@@ -267,9 +265,6 @@ void CameraState::run() {
 
     // Send the message
     pm.send(camera.cc.publish_name, msg);
-    if (camera.cc.stream_type == VISION_STREAM_ROAD && cnt % 100 == 3) {
-      publish_thumbnail(&pm, &camera.buf);  // this takes 10ms???
-    }
   }
 }
 

system/loggerd/SConscript

@@ -4,7 +4,7 @@ libs = [common, messaging, visionipc,
         'avformat', 'avcodec', 'avutil',
         'yuv', 'OpenCL', 'pthread', 'zstd']
 
-src = ['logger.cc', 'zstd_writer.cc', 'video_writer.cc', 'encoder/encoder.cc', 'encoder/v4l_encoder.cc']
+src = ['logger.cc', 'zstd_writer.cc', 'video_writer.cc', 'encoder/encoder.cc', 'encoder/v4l_encoder.cc', 'encoder/jpeg_encoder.cc']
 if arch != "larch64":
   src += ['encoder/ffmpeg_encoder.cc']
 
@@ -19,7 +19,7 @@ logger_lib = env.Library('logger', src)
 libs.insert(0, logger_lib)
 
 env.Program('loggerd', ['loggerd.cc'], LIBS=libs)
-env.Program('encoderd', ['encoderd.cc'], LIBS=libs)
+env.Program('encoderd', ['encoderd.cc'], LIBS=libs + ["jpeg"])
 env.Program('bootlog.cc', LIBS=libs)
 
 if GetOption('extras'):

system/loggerd/encoder/encoder.cc

@@ -6,12 +6,7 @@ VideoEncoder::VideoEncoder(const EncoderInfo &encoder_info, int in_width, int in
   out_width = encoder_info.frame_width > 0 ? encoder_info.frame_width : in_width;
   out_height = encoder_info.frame_height > 0 ? encoder_info.frame_height : in_height;
 
-
+  pm.reset(new PubMaster(std::vector{encoder_info.publish_name}));
-  std::vector pubs = {encoder_info.publish_name};
-  if (encoder_info.thumbnail_name != NULL) {
-    pubs.push_back(encoder_info.thumbnail_name);
-  }
-  pm.reset(new PubMaster(pubs));
 }
 
 void VideoEncoder::publisher_publish(int segment_num, uint32_t idx, VisionIpcBufExtra &extra,
@@ -45,15 +40,4 @@ void VideoEncoder::publisher_publish(int segment_num, uint32_t idx, VisionIpcBuf
   kj::ArrayOutputStream output_stream(kj::ArrayPtr<capnp::byte>(msg_cache.data(), bytes_size));
   capnp::writeMessage(output_stream, msg);
   pm->send(encoder_info.publish_name, msg_cache.data(), bytes_size);
-
-  // Publish keyframe thumbnail
-  if ((flags & V4L2_BUF_FLAG_KEYFRAME) && encoder_info.thumbnail_name != NULL) {
-    MessageBuilder tm;
-    auto thumbnail = tm.initEvent().initThumbnail();
-    thumbnail.setFrameId(extra.frame_id);
-    thumbnail.setTimestampEof(extra.timestamp_eof);
-    thumbnail.setThumbnail(dat);
-    thumbnail.setEncoding(cereal::Thumbnail::Encoding::KEYFRAME);
-    pm->send(encoder_info.thumbnail_name, tm);
-  }
 }

system/loggerd/encoder/encoder.h

@@ -30,8 +30,6 @@ public:
   void publisher_publish(int segment_num, uint32_t idx, VisionIpcBufExtra &extra, unsigned int flags, kj::ArrayPtr<capnp::byte> header, kj::ArrayPtr<capnp::byte> dat);
 
 protected:
-  void publish_thumbnail(uint32_t frame_id, uint64_t timestamp_eof, kj::ArrayPtr<capnp::byte> dat);
-
   int in_width, in_height;
   int out_width, out_height;
   const EncoderInfo encoder_info;

system/loggerd/encoder/jpeg_encoder.cc

@@ -0,0 +1,105 @@
+#include "system/loggerd/encoder/jpeg_encoder.h"
+
+#include <cassert>
+#include <cstring>
+
+JpegEncoder::JpegEncoder(const std::string &pusblish_name, int width, int height)
+    : publish_name(pusblish_name), thumbnail_width(width), thumbnail_height(height) {
+  yuv_buffer.resize((thumbnail_width * ((thumbnail_height + 15) & ~15) * 3) / 2);
+  pm = std::make_unique<PubMaster>(std::vector{pusblish_name.c_str()});
+}
+
+JpegEncoder::~JpegEncoder() {
+  if (out_buffer) {
+    free(out_buffer);
+  }
+}
+
+void JpegEncoder::pushThumbnail(VisionBuf *buf, const VisionIpcBufExtra &extra) {
+  generateThumbnail(buf->y, buf->uv, buf->width, buf->height, buf->stride);
+
+  MessageBuilder msg;
+  auto thumbnaild = msg.initEvent().initThumbnail();
+  thumbnaild.setFrameId(extra.frame_id);
+  thumbnaild.setTimestampEof(extra.timestamp_eof);
+  thumbnaild.setThumbnail({out_buffer, out_size});
+
+  pm->send(publish_name.c_str(), msg);
+}
+
+void JpegEncoder::generateThumbnail(const uint8_t *y_addr, const uint8_t *uv_addr, int width, int height, int stride) {
+  int downscale = width / thumbnail_width;
+  assert(downscale * thumbnail_height == height);
+
+  // make the buffer big enough. jpeg_write_raw_data requires 16-pixels aligned height to be used.
+  uint8_t *y_plane = yuv_buffer.data();
+  uint8_t *u_plane = y_plane + thumbnail_width * thumbnail_height;
+  uint8_t *v_plane = u_plane + (thumbnail_width * thumbnail_height) / 4;
+  {
+    // subsampled conversion from nv12 to yuv
+    for (int hy = 0; hy < thumbnail_height / 2; hy++) {
+      for (int hx = 0; hx < thumbnail_width / 2; hx++) {
+        int ix = hx * downscale + (downscale - 1) / 2;
+        int iy = hy * downscale + (downscale - 1) / 2;
+        y_plane[(hy * 2 + 0) * thumbnail_width + (hx * 2 + 0)] = y_addr[(iy * 2 + 0) * stride + ix * 2 + 0];
+        y_plane[(hy * 2 + 0) * thumbnail_width + (hx * 2 + 1)] = y_addr[(iy * 2 + 0) * stride + ix * 2 + 1];
+        y_plane[(hy * 2 + 1) * thumbnail_width + (hx * 2 + 0)] = y_addr[(iy * 2 + 1) * stride + ix * 2 + 0];
+        y_plane[(hy * 2 + 1) * thumbnail_width + (hx * 2 + 1)] = y_addr[(iy * 2 + 1) * stride + ix * 2 + 1];
+        u_plane[hy * thumbnail_width / 2 + hx] = uv_addr[iy * stride + ix * 2 + 0];
+        v_plane[hy * thumbnail_width / 2 + hx] = uv_addr[iy * stride + ix * 2 + 1];
+      }
+    }
+  }
+
+  compressToJpeg(y_plane, u_plane, v_plane);
+}
+
+void JpegEncoder::compressToJpeg(uint8_t *y_plane, uint8_t *u_plane, uint8_t *v_plane) {
+  struct jpeg_compress_struct cinfo;
+  struct jpeg_error_mgr jerr;
+  cinfo.err = jpeg_std_error(&jerr);
+  jpeg_create_compress(&cinfo);
+
+  if (out_buffer) {
+    free(out_buffer);
+    out_buffer = nullptr;
+    out_size = 0;
+  }
+  jpeg_mem_dest(&cinfo, &out_buffer, &out_size);
+
+  cinfo.image_width = thumbnail_width;
+  cinfo.image_height = thumbnail_height;
+  cinfo.input_components = 3;
+
+  jpeg_set_defaults(&cinfo);
+  jpeg_set_colorspace(&cinfo, JCS_YCbCr);
+  // configure sampling factors for yuv420.
+  cinfo.comp_info[0].h_samp_factor = 2;  // Y
+  cinfo.comp_info[0].v_samp_factor = 2;
+  cinfo.comp_info[1].h_samp_factor = 1;  // U
+  cinfo.comp_info[1].v_samp_factor = 1;
+  cinfo.comp_info[2].h_samp_factor = 1;  // V
+  cinfo.comp_info[2].v_samp_factor = 1;
+  cinfo.raw_data_in = TRUE;
+
+  jpeg_set_quality(&cinfo, 50, TRUE);
+  jpeg_start_compress(&cinfo, TRUE);
+
+  JSAMPROW y[16], u[8], v[8];
+  JSAMPARRAY planes[3]{y, u, v};
+
+  for (int line = 0; line < cinfo.image_height; line += 16) {
+    for (int i = 0; i < 16; ++i) {
+      y[i] = y_plane + (line + i) * cinfo.image_width;
+      if (i % 2 == 0) {
+        int offset = (cinfo.image_width / 2) * ((i + line) / 2);
+        u[i / 2] = u_plane + offset;
+        v[i / 2] = v_plane + offset;
+      }
+    }
+    jpeg_write_raw_data(&cinfo, planes, 16);
+  }
+
+  jpeg_finish_compress(&cinfo);
+  jpeg_destroy_compress(&cinfo);
+}

system/loggerd/encoder/jpeg_encoder.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <cstdio>
+#include <cstdlib>
+#include <cstddef>
+#include <cstdint>
+#include <jpeglib.h>
+#include <vector>
+#include <memory>
+#include "cereal/messaging/messaging.h"
+#include "msgq/visionipc/visionbuf.h"
+
+class JpegEncoder {
+public:
+  JpegEncoder(const std::string &pusblish_name, int width, int height);
+  ~JpegEncoder();
+  void pushThumbnail(VisionBuf *buf, const VisionIpcBufExtra &extra);
+
+private:
+  void generateThumbnail(const uint8_t *y, const uint8_t *uv, int width, int height, int stride);
+  void compressToJpeg(uint8_t *y_plane, uint8_t *u_plane, uint8_t *v_plane);
+
+  int thumbnail_width;
+  int thumbnail_height;
+  std::string publish_name;
+  std::vector<uint8_t> yuv_buffer;
+  std::unique_ptr<PubMaster> pm;
+
+  // JPEG output buffer
+  unsigned char* out_buffer = nullptr;
+  unsigned long out_size = 0;
+};

system/loggerd/encoderd.cc

@@ -1,6 +1,7 @@
 #include <cassert>
 
 #include "system/loggerd/loggerd.h"
+#include "system/loggerd/encoder/jpeg_encoder.h"
 
 #ifdef QCOM2
 #include "system/loggerd/encoder/v4l_encoder.h"
@@ -50,6 +51,8 @@ void encoder_thread(EncoderdState *s, const LogCameraInfo &cam_info) {
   std::vector<std::unique_ptr<Encoder>> encoders;
   VisionIpcClient vipc_client = VisionIpcClient("camerad", cam_info.stream_type, false);
 
+  std::unique_ptr<JpegEncoder> jpeg_encoder;
+
   int cur_seg = 0;
   while (!do_exit) {
     if (!vipc_client.connect(false)) {
@@ -67,6 +70,11 @@ void encoder_thread(EncoderdState *s, const LogCameraInfo &cam_info) {
         auto &e = encoders.emplace_back(new Encoder(encoder_info, buf_info.width, buf_info.height));
         e->encoder_open(nullptr);
       }
+
+      // Only one thumbnail can be generated per camera stream
+      if (auto thumbnail_name = cam_info.encoder_infos[0].thumbnail_name) {
+        jpeg_encoder = std::make_unique<JpegEncoder>(thumbnail_name, buf_info.width / 4, buf_info.height / 4);
+      }
     }
 
     bool lagging = false;
@@ -108,6 +116,10 @@ void encoder_thread(EncoderdState *s, const LogCameraInfo &cam_info) {
           LOGE("Failed to encode frame. frame_id: %d", extra.frame_id);
         }
       }
+
+      if (jpeg_encoder && (extra.frame_id % 1200 == 100)) {
+        jpeg_encoder->pushThumbnail(buf, extra);
+      }
     }
   }
 }

system/loggerd/loggerd.h

@@ -56,6 +56,7 @@ public:
 
 const EncoderInfo main_road_encoder_info = {
   .publish_name = "roadEncodeData",
+  .thumbnail_name = "thumbnail",
   .filename = "fcamera.hevc",
   INIT_ENCODE_FUNCTIONS(RoadEncode),
 };

tools/install_ubuntu_dependencies.sh

@@ -45,6 +45,7 @@ function install_ubuntu_common_requirements() {
     libgles2-mesa-dev \
     libglfw3-dev \
     libglib2.0-0 \
+    libjpeg-dev \
     libqt5charts5-dev \
     libncurses5-dev \
     libssl-dev \