From 3c0990e7162c47d9a6e6afbd10ea4a1d28f09afe Mon Sep 17 00:00:00 2001 From: Dean Lee Date: Tue, 11 Feb 2025 07:43:46 +0800 Subject: [PATCH] move thumbnail generation from camerad to encoderd (#34554) Co-authored-by: Cameron Clough --- cereal/services.py | 2 +- system/camerad/SConscript | 2 +- system/camerad/cameras/camera_common.cc | 91 -------------------- system/camerad/cameras/camera_common.h | 1 - system/camerad/cameras/camera_qcom2.cc | 7 +- system/loggerd/SConscript | 4 +- system/loggerd/encoder/encoder.cc | 18 +--- system/loggerd/encoder/encoder.h | 2 - system/loggerd/encoder/jpeg_encoder.cc | 105 ++++++++++++++++++++++++ system/loggerd/encoder/jpeg_encoder.h | 32 ++++++++ system/loggerd/encoderd.cc | 12 +++ system/loggerd/loggerd.h | 1 + tools/install_ubuntu_dependencies.sh | 1 + 13 files changed, 157 insertions(+), 121 deletions(-) create mode 100644 system/loggerd/encoder/jpeg_encoder.cc create mode 100644 system/loggerd/encoder/jpeg_encoder.h diff --git a/cereal/services.py b/cereal/services.py index 87fdca77b7..aad83177bb 100755 --- a/cereal/services.py +++ b/cereal/services.py @@ -53,7 +53,7 @@ _services: dict[str, tuple] = { "livePose": (True, 20., 4), "liveParameters": (True, 20., 5), "cameraOdometry": (True, 20., 10), - "thumbnail": (True, 0.2, 1), + "thumbnail": (True, 1 / 60., 1), "onroadEvents": (True, 1., 1), "carParams": (True, 0.02, 1), "roadCameraState": (True, 20., 20), diff --git a/system/camerad/SConscript b/system/camerad/SConscript index 46eacf94ef..0d936176b9 100644 --- a/system/camerad/SConscript +++ b/system/camerad/SConscript @@ -1,6 +1,6 @@ Import('env', 'arch', 'messaging', 'common', 'gpucommon', 'visionipc') -libs = ['pthread', common, 'jpeg', 'OpenCL', messaging, visionipc, gpucommon] +libs = ['pthread', common, 'OpenCL', messaging, visionipc, gpucommon] if arch != "Darwin": camera_obj = env.Object(['cameras/camera_qcom2.cc', 'cameras/camera_common.cc', 'cameras/spectra.cc', diff --git a/system/camerad/cameras/camera_common.cc b/system/camerad/cameras/camera_common.cc index 2a218affa8..666d027beb 100644 --- a/system/camerad/cameras/camera_common.cc +++ b/system/camerad/cameras/camera_common.cc @@ -3,8 +3,6 @@ #include #include -#include - #include "common/clutil.h" #include "common/swaglog.h" @@ -150,95 +148,6 @@ kj::Array get_raw_frame_image(const CameraBuf *b) { return kj::mv(frame_image); } -static kj::Array yuv420_to_jpeg(const CameraBuf *b, int thumbnail_width, int thumbnail_height) { - int downscale = b->cur_yuv_buf->width / thumbnail_width; - assert(downscale * thumbnail_height == b->cur_yuv_buf->height); - int in_stride = b->cur_yuv_buf->stride; - - // make the buffer big enough. jpeg_write_raw_data requires 16-pixels aligned height to be used. - std::unique_ptr buf(new uint8_t[(thumbnail_width * ((thumbnail_height + 15) & ~15) * 3) / 2]); - uint8_t *y_plane = buf.get(); - uint8_t *u_plane = y_plane + thumbnail_width * thumbnail_height; - uint8_t *v_plane = u_plane + (thumbnail_width * thumbnail_height) / 4; - { - // subsampled conversion from nv12 to yuv - for (int hy = 0; hy < thumbnail_height/2; hy++) { - for (int hx = 0; hx < thumbnail_width/2; hx++) { - int ix = hx * downscale + (downscale-1)/2; - int iy = hy * downscale + (downscale-1)/2; - y_plane[(hy*2 + 0)*thumbnail_width + (hx*2 + 0)] = b->cur_yuv_buf->y[(iy*2 + 0) * in_stride + ix*2 + 0]; - y_plane[(hy*2 + 0)*thumbnail_width + (hx*2 + 1)] = b->cur_yuv_buf->y[(iy*2 + 0) * in_stride + ix*2 + 1]; - y_plane[(hy*2 + 1)*thumbnail_width + (hx*2 + 0)] = b->cur_yuv_buf->y[(iy*2 + 1) * in_stride + ix*2 + 0]; - y_plane[(hy*2 + 1)*thumbnail_width + (hx*2 + 1)] = b->cur_yuv_buf->y[(iy*2 + 1) * in_stride + ix*2 + 1]; - u_plane[hy*thumbnail_width/2 + hx] = b->cur_yuv_buf->uv[iy*in_stride + ix*2 + 0]; - v_plane[hy*thumbnail_width/2 + hx] = b->cur_yuv_buf->uv[iy*in_stride + ix*2 + 1]; - } - } - } - - struct jpeg_compress_struct cinfo; - struct jpeg_error_mgr jerr; - cinfo.err = jpeg_std_error(&jerr); - jpeg_create_compress(&cinfo); - - uint8_t *thumbnail_buffer = nullptr; - size_t thumbnail_len = 0; - jpeg_mem_dest(&cinfo, &thumbnail_buffer, &thumbnail_len); - - cinfo.image_width = thumbnail_width; - cinfo.image_height = thumbnail_height; - cinfo.input_components = 3; - - jpeg_set_defaults(&cinfo); - jpeg_set_colorspace(&cinfo, JCS_YCbCr); - // configure sampling factors for yuv420. - cinfo.comp_info[0].h_samp_factor = 2; // Y - cinfo.comp_info[0].v_samp_factor = 2; - cinfo.comp_info[1].h_samp_factor = 1; // U - cinfo.comp_info[1].v_samp_factor = 1; - cinfo.comp_info[2].h_samp_factor = 1; // V - cinfo.comp_info[2].v_samp_factor = 1; - cinfo.raw_data_in = TRUE; - - jpeg_set_quality(&cinfo, 50, TRUE); - jpeg_start_compress(&cinfo, TRUE); - - JSAMPROW y[16], u[8], v[8]; - JSAMPARRAY planes[3]{y, u, v}; - - for (int line = 0; line < cinfo.image_height; line += 16) { - for (int i = 0; i < 16; ++i) { - y[i] = y_plane + (line + i) * cinfo.image_width; - if (i % 2 == 0) { - int offset = (cinfo.image_width / 2) * ((i + line) / 2); - u[i / 2] = u_plane + offset; - v[i / 2] = v_plane + offset; - } - } - jpeg_write_raw_data(&cinfo, planes, 16); - } - - jpeg_finish_compress(&cinfo); - jpeg_destroy_compress(&cinfo); - - kj::Array dat = kj::heapArray(thumbnail_buffer, thumbnail_len); - free(thumbnail_buffer); - return dat; -} - -void publish_thumbnail(PubMaster *pm, const CameraBuf *b) { - auto thumbnail = yuv420_to_jpeg(b, b->out_img_width / 4, b->out_img_height / 4); - if (thumbnail.size() == 0) return; - - MessageBuilder msg; - auto thumbnaild = msg.initEvent().initThumbnail(); - thumbnaild.setFrameId(b->cur_frame_data.frame_id); - thumbnaild.setTimestampEof(b->cur_frame_data.timestamp_eof); - thumbnaild.setThumbnail(thumbnail); - - pm->send("thumbnail", msg); -} - float set_exposure_target(const CameraBuf *b, Rect ae_xywh, int x_skip, int y_skip) { int lum_med; uint32_t lum_binning[256] = {0}; diff --git a/system/camerad/cameras/camera_common.h b/system/camerad/cameras/camera_common.h index 768626737a..242e17ef25 100644 --- a/system/camerad/cameras/camera_common.h +++ b/system/camerad/cameras/camera_common.h @@ -52,5 +52,4 @@ public: void camerad_thread(); kj::Array get_raw_frame_image(const CameraBuf *b); float set_exposure_target(const CameraBuf *b, Rect ae_xywh, int x_skip, int y_skip); -void publish_thumbnail(PubMaster *pm, const CameraBuf *b); int open_v4l_by_name_and_index(const char name[], int index = 0, int flags = O_RDWR | O_NONBLOCK); diff --git a/system/camerad/cameras/camera_qcom2.cc b/system/camerad/cameras/camera_qcom2.cc index 4c22ad527c..e5c9f548ab 100644 --- a/system/camerad/cameras/camera_qcom2.cc +++ b/system/camerad/cameras/camera_qcom2.cc @@ -231,9 +231,7 @@ void CameraState::set_camera_exposure(float grey_frac) { void CameraState::run() { util::set_thread_name(camera.cc.publish_name); - std::vector pubs = {camera.cc.publish_name}; - if (camera.cc.stream_type == VISION_STREAM_ROAD) pubs.push_back("thumbnail"); - PubMaster pm(pubs); + PubMaster pm(std::vector{camera.cc.publish_name}); for (uint32_t cnt = 0; !do_exit; ++cnt) { // Acquire the buffer; continue if acquisition fails @@ -267,9 +265,6 @@ void CameraState::run() { // Send the message pm.send(camera.cc.publish_name, msg); - if (camera.cc.stream_type == VISION_STREAM_ROAD && cnt % 100 == 3) { - publish_thumbnail(&pm, &camera.buf); // this takes 10ms??? - } } } diff --git a/system/loggerd/SConscript b/system/loggerd/SConscript index 9b5daf6955..cf169f4dc6 100644 --- a/system/loggerd/SConscript +++ b/system/loggerd/SConscript @@ -4,7 +4,7 @@ libs = [common, messaging, visionipc, 'avformat', 'avcodec', 'avutil', 'yuv', 'OpenCL', 'pthread', 'zstd'] -src = ['logger.cc', 'zstd_writer.cc', 'video_writer.cc', 'encoder/encoder.cc', 'encoder/v4l_encoder.cc'] +src = ['logger.cc', 'zstd_writer.cc', 'video_writer.cc', 'encoder/encoder.cc', 'encoder/v4l_encoder.cc', 'encoder/jpeg_encoder.cc'] if arch != "larch64": src += ['encoder/ffmpeg_encoder.cc'] @@ -19,7 +19,7 @@ logger_lib = env.Library('logger', src) libs.insert(0, logger_lib) env.Program('loggerd', ['loggerd.cc'], LIBS=libs) -env.Program('encoderd', ['encoderd.cc'], LIBS=libs) +env.Program('encoderd', ['encoderd.cc'], LIBS=libs + ["jpeg"]) env.Program('bootlog.cc', LIBS=libs) if GetOption('extras'): diff --git a/system/loggerd/encoder/encoder.cc b/system/loggerd/encoder/encoder.cc index e9fdc9c98c..366a4b961e 100644 --- a/system/loggerd/encoder/encoder.cc +++ b/system/loggerd/encoder/encoder.cc @@ -6,12 +6,7 @@ VideoEncoder::VideoEncoder(const EncoderInfo &encoder_info, int in_width, int in out_width = encoder_info.frame_width > 0 ? encoder_info.frame_width : in_width; out_height = encoder_info.frame_height > 0 ? encoder_info.frame_height : in_height; - - std::vector pubs = {encoder_info.publish_name}; - if (encoder_info.thumbnail_name != NULL) { - pubs.push_back(encoder_info.thumbnail_name); - } - pm.reset(new PubMaster(pubs)); + pm.reset(new PubMaster(std::vector{encoder_info.publish_name})); } void VideoEncoder::publisher_publish(int segment_num, uint32_t idx, VisionIpcBufExtra &extra, @@ -45,15 +40,4 @@ void VideoEncoder::publisher_publish(int segment_num, uint32_t idx, VisionIpcBuf kj::ArrayOutputStream output_stream(kj::ArrayPtr(msg_cache.data(), bytes_size)); capnp::writeMessage(output_stream, msg); pm->send(encoder_info.publish_name, msg_cache.data(), bytes_size); - - // Publish keyframe thumbnail - if ((flags & V4L2_BUF_FLAG_KEYFRAME) && encoder_info.thumbnail_name != NULL) { - MessageBuilder tm; - auto thumbnail = tm.initEvent().initThumbnail(); - thumbnail.setFrameId(extra.frame_id); - thumbnail.setTimestampEof(extra.timestamp_eof); - thumbnail.setThumbnail(dat); - thumbnail.setEncoding(cereal::Thumbnail::Encoding::KEYFRAME); - pm->send(encoder_info.thumbnail_name, tm); - } } diff --git a/system/loggerd/encoder/encoder.h b/system/loggerd/encoder/encoder.h index 3863f89744..d94904f106 100644 --- a/system/loggerd/encoder/encoder.h +++ b/system/loggerd/encoder/encoder.h @@ -30,8 +30,6 @@ public: void publisher_publish(int segment_num, uint32_t idx, VisionIpcBufExtra &extra, unsigned int flags, kj::ArrayPtr header, kj::ArrayPtr dat); protected: - void publish_thumbnail(uint32_t frame_id, uint64_t timestamp_eof, kj::ArrayPtr dat); - int in_width, in_height; int out_width, out_height; const EncoderInfo encoder_info; diff --git a/system/loggerd/encoder/jpeg_encoder.cc b/system/loggerd/encoder/jpeg_encoder.cc new file mode 100644 index 0000000000..6bb946157c --- /dev/null +++ b/system/loggerd/encoder/jpeg_encoder.cc @@ -0,0 +1,105 @@ +#include "system/loggerd/encoder/jpeg_encoder.h" + +#include +#include + +JpegEncoder::JpegEncoder(const std::string &pusblish_name, int width, int height) + : publish_name(pusblish_name), thumbnail_width(width), thumbnail_height(height) { + yuv_buffer.resize((thumbnail_width * ((thumbnail_height + 15) & ~15) * 3) / 2); + pm = std::make_unique(std::vector{pusblish_name.c_str()}); +} + +JpegEncoder::~JpegEncoder() { + if (out_buffer) { + free(out_buffer); + } +} + +void JpegEncoder::pushThumbnail(VisionBuf *buf, const VisionIpcBufExtra &extra) { + generateThumbnail(buf->y, buf->uv, buf->width, buf->height, buf->stride); + + MessageBuilder msg; + auto thumbnaild = msg.initEvent().initThumbnail(); + thumbnaild.setFrameId(extra.frame_id); + thumbnaild.setTimestampEof(extra.timestamp_eof); + thumbnaild.setThumbnail({out_buffer, out_size}); + + pm->send(publish_name.c_str(), msg); +} + +void JpegEncoder::generateThumbnail(const uint8_t *y_addr, const uint8_t *uv_addr, int width, int height, int stride) { + int downscale = width / thumbnail_width; + assert(downscale * thumbnail_height == height); + + // make the buffer big enough. jpeg_write_raw_data requires 16-pixels aligned height to be used. + uint8_t *y_plane = yuv_buffer.data(); + uint8_t *u_plane = y_plane + thumbnail_width * thumbnail_height; + uint8_t *v_plane = u_plane + (thumbnail_width * thumbnail_height) / 4; + { + // subsampled conversion from nv12 to yuv + for (int hy = 0; hy < thumbnail_height / 2; hy++) { + for (int hx = 0; hx < thumbnail_width / 2; hx++) { + int ix = hx * downscale + (downscale - 1) / 2; + int iy = hy * downscale + (downscale - 1) / 2; + y_plane[(hy * 2 + 0) * thumbnail_width + (hx * 2 + 0)] = y_addr[(iy * 2 + 0) * stride + ix * 2 + 0]; + y_plane[(hy * 2 + 0) * thumbnail_width + (hx * 2 + 1)] = y_addr[(iy * 2 + 0) * stride + ix * 2 + 1]; + y_plane[(hy * 2 + 1) * thumbnail_width + (hx * 2 + 0)] = y_addr[(iy * 2 + 1) * stride + ix * 2 + 0]; + y_plane[(hy * 2 + 1) * thumbnail_width + (hx * 2 + 1)] = y_addr[(iy * 2 + 1) * stride + ix * 2 + 1]; + u_plane[hy * thumbnail_width / 2 + hx] = uv_addr[iy * stride + ix * 2 + 0]; + v_plane[hy * thumbnail_width / 2 + hx] = uv_addr[iy * stride + ix * 2 + 1]; + } + } + } + + compressToJpeg(y_plane, u_plane, v_plane); +} + +void JpegEncoder::compressToJpeg(uint8_t *y_plane, uint8_t *u_plane, uint8_t *v_plane) { + struct jpeg_compress_struct cinfo; + struct jpeg_error_mgr jerr; + cinfo.err = jpeg_std_error(&jerr); + jpeg_create_compress(&cinfo); + + if (out_buffer) { + free(out_buffer); + out_buffer = nullptr; + out_size = 0; + } + jpeg_mem_dest(&cinfo, &out_buffer, &out_size); + + cinfo.image_width = thumbnail_width; + cinfo.image_height = thumbnail_height; + cinfo.input_components = 3; + + jpeg_set_defaults(&cinfo); + jpeg_set_colorspace(&cinfo, JCS_YCbCr); + // configure sampling factors for yuv420. + cinfo.comp_info[0].h_samp_factor = 2; // Y + cinfo.comp_info[0].v_samp_factor = 2; + cinfo.comp_info[1].h_samp_factor = 1; // U + cinfo.comp_info[1].v_samp_factor = 1; + cinfo.comp_info[2].h_samp_factor = 1; // V + cinfo.comp_info[2].v_samp_factor = 1; + cinfo.raw_data_in = TRUE; + + jpeg_set_quality(&cinfo, 50, TRUE); + jpeg_start_compress(&cinfo, TRUE); + + JSAMPROW y[16], u[8], v[8]; + JSAMPARRAY planes[3]{y, u, v}; + + for (int line = 0; line < cinfo.image_height; line += 16) { + for (int i = 0; i < 16; ++i) { + y[i] = y_plane + (line + i) * cinfo.image_width; + if (i % 2 == 0) { + int offset = (cinfo.image_width / 2) * ((i + line) / 2); + u[i / 2] = u_plane + offset; + v[i / 2] = v_plane + offset; + } + } + jpeg_write_raw_data(&cinfo, planes, 16); + } + + jpeg_finish_compress(&cinfo); + jpeg_destroy_compress(&cinfo); +} diff --git a/system/loggerd/encoder/jpeg_encoder.h b/system/loggerd/encoder/jpeg_encoder.h new file mode 100644 index 0000000000..af1427c19a --- /dev/null +++ b/system/loggerd/encoder/jpeg_encoder.h @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include "cereal/messaging/messaging.h" +#include "msgq/visionipc/visionbuf.h" + +class JpegEncoder { +public: + JpegEncoder(const std::string &pusblish_name, int width, int height); + ~JpegEncoder(); + void pushThumbnail(VisionBuf *buf, const VisionIpcBufExtra &extra); + +private: + void generateThumbnail(const uint8_t *y, const uint8_t *uv, int width, int height, int stride); + void compressToJpeg(uint8_t *y_plane, uint8_t *u_plane, uint8_t *v_plane); + + int thumbnail_width; + int thumbnail_height; + std::string publish_name; + std::vector yuv_buffer; + std::unique_ptr pm; + + // JPEG output buffer + unsigned char* out_buffer = nullptr; + unsigned long out_size = 0; +}; diff --git a/system/loggerd/encoderd.cc b/system/loggerd/encoderd.cc index feb9a921ec..c777e6eb7a 100644 --- a/system/loggerd/encoderd.cc +++ b/system/loggerd/encoderd.cc @@ -1,6 +1,7 @@ #include #include "system/loggerd/loggerd.h" +#include "system/loggerd/encoder/jpeg_encoder.h" #ifdef QCOM2 #include "system/loggerd/encoder/v4l_encoder.h" @@ -50,6 +51,8 @@ void encoder_thread(EncoderdState *s, const LogCameraInfo &cam_info) { std::vector> encoders; VisionIpcClient vipc_client = VisionIpcClient("camerad", cam_info.stream_type, false); + std::unique_ptr jpeg_encoder; + int cur_seg = 0; while (!do_exit) { if (!vipc_client.connect(false)) { @@ -67,6 +70,11 @@ void encoder_thread(EncoderdState *s, const LogCameraInfo &cam_info) { auto &e = encoders.emplace_back(new Encoder(encoder_info, buf_info.width, buf_info.height)); e->encoder_open(nullptr); } + + // Only one thumbnail can be generated per camera stream + if (auto thumbnail_name = cam_info.encoder_infos[0].thumbnail_name) { + jpeg_encoder = std::make_unique(thumbnail_name, buf_info.width / 4, buf_info.height / 4); + } } bool lagging = false; @@ -108,6 +116,10 @@ void encoder_thread(EncoderdState *s, const LogCameraInfo &cam_info) { LOGE("Failed to encode frame. frame_id: %d", extra.frame_id); } } + + if (jpeg_encoder && (extra.frame_id % 1200 == 100)) { + jpeg_encoder->pushThumbnail(buf, extra); + } } } } diff --git a/system/loggerd/loggerd.h b/system/loggerd/loggerd.h index e8cdadda06..27d2d37fc4 100644 --- a/system/loggerd/loggerd.h +++ b/system/loggerd/loggerd.h @@ -56,6 +56,7 @@ public: const EncoderInfo main_road_encoder_info = { .publish_name = "roadEncodeData", + .thumbnail_name = "thumbnail", .filename = "fcamera.hevc", INIT_ENCODE_FUNCTIONS(RoadEncode), }; diff --git a/tools/install_ubuntu_dependencies.sh b/tools/install_ubuntu_dependencies.sh index a48160d8f8..150999fe7c 100755 --- a/tools/install_ubuntu_dependencies.sh +++ b/tools/install_ubuntu_dependencies.sh @@ -45,6 +45,7 @@ function install_ubuntu_common_requirements() { libgles2-mesa-dev \ libglfw3-dev \ libglib2.0-0 \ + libjpeg-dev \ libqt5charts5-dev \ libncurses5-dev \ libssl-dev \