From cd4e070b479e0e3df5545ac37c47d93481ecfa7c Mon Sep 17 00:00:00 2001 From: Dean Lee Date: Sat, 18 Sep 2021 08:00:21 +0800 Subject: [PATCH] camerad: use the YUV buffer to create the jpeg thumbnail (#21936) * yuv420_to_jpeg continue * add comments * cleanup * return ky::array old-commit-hash: e0b5b4573ed34890a5e923012bf96fe311fd11dd --- selfdrive/camerad/SConscript | 2 +- selfdrive/camerad/cameras/camera_common.cc | 94 +++++++++++++--------- 2 files changed, 57 insertions(+), 39 deletions(-) diff --git a/selfdrive/camerad/SConscript b/selfdrive/camerad/SConscript index c12d254074..4e20c16dc4 100644 --- a/selfdrive/camerad/SConscript +++ b/selfdrive/camerad/SConscript @@ -1,6 +1,6 @@ Import('env', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc', 'USE_WEBCAM') -libs = ['m', 'pthread', common, 'jpeg', 'OpenCL', cereal, messaging, 'zmq', 'capnp', 'kj', visionipc, gpucommon] +libs = ['m', 'pthread', common, 'jpeg', 'OpenCL', 'yuv', cereal, messaging, 'zmq', 'capnp', 'kj', visionipc, gpucommon] if arch == "aarch64": libs += ['gsl', 'CB', 'adreno_utils', 'EGL', 'GLESv3', 'cutils', 'ui'] diff --git a/selfdrive/camerad/cameras/camera_common.cc b/selfdrive/camerad/cameras/camera_common.cc index 89052e5e14..1cd608addd 100644 --- a/selfdrive/camerad/cameras/camera_common.cc +++ b/selfdrive/camerad/cameras/camera_common.cc @@ -216,66 +216,84 @@ kj::Array get_frame_image(const CameraBuf *b) { return kj::mv(frame_image); } -static void publish_thumbnail(PubMaster *pm, const CameraBuf *b) { - uint8_t* thumbnail_buffer = NULL; - unsigned long thumbnail_len = 0; - - unsigned char *row = (unsigned char *)malloc(b->rgb_width/4*3); +static kj::Array yuv420_to_jpeg(const CameraBuf *b, int thumbnail_width, int thumbnail_height) { + std::unique_ptr buf(new uint8_t[(thumbnail_width * thumbnail_height * 3) / 2]); + uint8_t *y_plane = buf.get(); + uint8_t *u_plane = y_plane + thumbnail_width * thumbnail_height; + uint8_t *v_plane = u_plane + (thumbnail_width * thumbnail_height) / 4; + { + int result = libyuv::I420Scale( + b->cur_yuv_buf->y, b->rgb_width, b->cur_yuv_buf->u, b->rgb_width / 2, b->cur_yuv_buf->v, b->rgb_width / 2, + b->rgb_width, b->rgb_height, + y_plane, thumbnail_width, u_plane, thumbnail_width / 2, v_plane, thumbnail_width / 2, + thumbnail_width, thumbnail_height, libyuv::kFilterNone); + if (result != 0) { + LOGE("Generate YUV thumbnail failed."); + return {}; + } + } struct jpeg_compress_struct cinfo; struct jpeg_error_mgr jerr; - cinfo.err = jpeg_std_error(&jerr); jpeg_create_compress(&cinfo); + + uint8_t *thumbnail_buffer = nullptr; + size_t thumbnail_len = 0; jpeg_mem_dest(&cinfo, &thumbnail_buffer, &thumbnail_len); - cinfo.image_width = b->rgb_width / 4; - cinfo.image_height = b->rgb_height / 4; + cinfo.image_width = thumbnail_width; + cinfo.image_height = thumbnail_height; cinfo.input_components = 3; - cinfo.in_color_space = JCS_RGB; jpeg_set_defaults(&cinfo); -#ifndef __APPLE__ - jpeg_set_quality(&cinfo, 50, true); - jpeg_start_compress(&cinfo, true); -#else - jpeg_set_quality(&cinfo, 50, static_cast(true) ); - jpeg_start_compress(&cinfo, static_cast(true) ); -#endif - - JSAMPROW row_pointer[1]; - const uint8_t *bgr_ptr = (const uint8_t *)b->cur_rgb_buf->addr; - for (int ii = 0; ii < b->rgb_height/4; ii+=1) { - for (int j = 0; j < b->rgb_width*3; j+=12) { - for (int k = 0; k < 3; k++) { - uint16_t dat = 0; - int i = ii * 4; - dat += bgr_ptr[b->rgb_stride*i + j + k]; - dat += bgr_ptr[b->rgb_stride*i + j+3 + k]; - dat += bgr_ptr[b->rgb_stride*(i+1) + j + k]; - dat += bgr_ptr[b->rgb_stride*(i+1) + j+3 + k]; - dat += bgr_ptr[b->rgb_stride*(i+2) + j + k]; - dat += bgr_ptr[b->rgb_stride*(i+2) + j+3 + k]; - dat += bgr_ptr[b->rgb_stride*(i+3) + j + k]; - dat += bgr_ptr[b->rgb_stride*(i+3) + j+3 + k]; - row[(j/4) + (2-k)] = dat/8; + jpeg_set_colorspace(&cinfo, JCS_YCbCr); + // configure sampling factors for yuv420. + cinfo.comp_info[0].h_samp_factor = 2; // Y + cinfo.comp_info[0].v_samp_factor = 2; + cinfo.comp_info[1].h_samp_factor = 1; // U + cinfo.comp_info[1].v_samp_factor = 1; + cinfo.comp_info[2].h_samp_factor = 1; // V + cinfo.comp_info[2].v_samp_factor = 1; + cinfo.raw_data_in = TRUE; + + jpeg_set_quality(&cinfo, 50, TRUE); + jpeg_start_compress(&cinfo, TRUE); + + JSAMPROW y[16], u[8], v[8]; + JSAMPARRAY planes[3]{y, u, v}; + + for (int line = 0; line < cinfo.image_height; line += 16) { + for (int i = 0; i < 16; ++i) { + y[i] = y_plane + (line + i) * cinfo.image_width; + if (i % 2 == 0) { + int offset = (cinfo.image_width / 2) * ((i + line) / 2); + u[i / 2] = u_plane + offset; + v[i / 2] = v_plane + offset; } } - row_pointer[0] = row; - jpeg_write_scanlines(&cinfo, row_pointer, 1); + jpeg_write_raw_data(&cinfo, planes, 16); } + jpeg_finish_compress(&cinfo); jpeg_destroy_compress(&cinfo); - free(row); + + kj::Array dat = kj::heapArray(thumbnail_buffer, thumbnail_len); + free(thumbnail_buffer); + return dat; +} + +static void publish_thumbnail(PubMaster *pm, const CameraBuf *b) { + auto thumbnail = yuv420_to_jpeg(b, b->rgb_width / 4, b->rgb_height / 4); + if (thumbnail.size() == 0) return; MessageBuilder msg; auto thumbnaild = msg.initEvent().initThumbnail(); thumbnaild.setFrameId(b->cur_frame_data.frame_id); thumbnaild.setTimestampEof(b->cur_frame_data.timestamp_eof); - thumbnaild.setThumbnail(kj::arrayPtr((const uint8_t*)thumbnail_buffer, thumbnail_len)); + thumbnaild.setThumbnail(thumbnail); pm->send("thumbnail", msg); - free(thumbnail_buffer); } float set_exposure_target(const CameraBuf *b, int x_start, int x_end, int x_skip, int y_start, int y_end, int y_skip) {