From 2008db47c208f6f0fd661cc45f215bb80aef38d0 Mon Sep 17 00:00:00 2001 From: George Hotz <72895+geohot@users.noreply.github.com> Date: Tue, 10 May 2022 15:20:51 -0700 Subject: [PATCH] encoderd: make work on PC (#24483) * don't use the codec in video_writer * this produces broken videos for some reason * bugfix * refactor on the class * works on device * fix codec * no codec enum * fix pc * move into dirs * these includes also * rename it ffmpegencoder * add avcodec_close Co-authored-by: Comma Device --- cereal | 2 +- release/files_common | 15 +-- selfdrive/loggerd/SConscript | 9 +- selfdrive/loggerd/encoder.h | 13 --- selfdrive/loggerd/encoder/encoder.cc | 79 ++++++++++++++++ selfdrive/loggerd/encoder/encoder.h | 60 ++++++++++++ .../ffmpeg_encoder.cc} | 58 ++++++++---- selfdrive/loggerd/encoder/ffmpeg_encoder.h | 38 ++++++++ .../loggerd/{ => encoder}/v4l_encoder.cc | 94 ++----------------- selfdrive/loggerd/{ => encoder}/v4l_encoder.h | 26 ++--- .../loggerd/{ => encoder}/video_writer.cc | 27 +++--- .../loggerd/{ => encoder}/video_writer.h | 6 +- selfdrive/loggerd/encoderd.cc | 6 +- selfdrive/loggerd/loggerd.cc | 8 +- selfdrive/loggerd/loggerd.h | 8 +- selfdrive/loggerd/raw_logger.h | 41 -------- selfdrive/loggerd/remote_encoder.cc | 2 +- selfdrive/loggerd/remote_encoder.h | 3 +- tools/camerastream/compressed_vipc.py | 2 +- 19 files changed, 279 insertions(+), 218 deletions(-) delete mode 100644 selfdrive/loggerd/encoder.h create mode 100644 selfdrive/loggerd/encoder/encoder.cc create mode 100644 selfdrive/loggerd/encoder/encoder.h rename selfdrive/loggerd/{raw_logger.cc => encoder/ffmpeg_encoder.cc} (58%) create mode 100644 selfdrive/loggerd/encoder/ffmpeg_encoder.h rename selfdrive/loggerd/{ => encoder}/v4l_encoder.cc (75%) rename selfdrive/loggerd/{ => encoder}/v4l_encoder.h (52%) rename selfdrive/loggerd/{ => encoder}/video_writer.cc (83%) rename selfdrive/loggerd/{ => encoder}/video_writer.h (80%) delete mode 100644 selfdrive/loggerd/raw_logger.h diff --git a/cereal b/cereal index c7d3a0acba..67e5c5ca37 160000 --- a/cereal +++ b/cereal @@ -1 +1 @@ -Subproject commit c7d3a0acbae267ef93d30044e1941e060dac9e48 +Subproject commit 67e5c5ca378fcf6db53e968e5ad9e4a98d32ed0f diff --git a/release/files_common b/release/files_common index c44f10229e..46f555fe0b 100644 --- a/release/files_common +++ b/release/files_common @@ -298,11 +298,12 @@ selfdrive/proclogd/proclog.cc selfdrive/proclogd/proclog.h selfdrive/loggerd/SConscript -selfdrive/loggerd/encoder.h -selfdrive/loggerd/v4l_encoder.cc -selfdrive/loggerd/v4l_encoder.h -selfdrive/loggerd/video_writer.cc -selfdrive/loggerd/video_writer.h +selfdrive/loggerd/encoder/encoder.cc +selfdrive/loggerd/encoder/encoder.h +selfdrive/loggerd/encoder/v4l_encoder.cc +selfdrive/loggerd/encoder/v4l_encoder.h +selfdrive/loggerd/encoder/video_writer.cc +selfdrive/loggerd/encoder/video_writer.h selfdrive/loggerd/remote_encoder.cc selfdrive/loggerd/remote_encoder.h selfdrive/loggerd/logger.cc @@ -312,8 +313,8 @@ selfdrive/loggerd/loggerd.h selfdrive/loggerd/encoderd.cc selfdrive/loggerd/main.cc selfdrive/loggerd/bootlog.cc -selfdrive/loggerd/raw_logger.cc -selfdrive/loggerd/raw_logger.h +selfdrive/loggerd/encoder/ffmpeg_encoder.cc +selfdrive/loggerd/encoder/ffmpeg_encoder.h selfdrive/loggerd/__init__.py selfdrive/loggerd/config.py diff --git a/selfdrive/loggerd/SConscript b/selfdrive/loggerd/SConscript index 7403dafb6d..81eca19a47 100644 --- a/selfdrive/loggerd/SConscript +++ b/selfdrive/loggerd/SConscript @@ -5,11 +5,11 @@ libs = [common, cereal, messaging, visionipc, 'avformat', 'avcodec', 'swscale', 'avutil', 'yuv', 'OpenCL', 'pthread'] -src = ['logger.cc', 'loggerd.cc', 'video_writer.cc', 'remote_encoder.cc'] +src = ['logger.cc', 'loggerd.cc', 'encoder/video_writer.cc', 'remote_encoder.cc', 'encoder/encoder.cc'] if arch == "larch64": - src += ['v4l_encoder.cc'] + src += ['encoder/v4l_encoder.cc'] else: - src += ['raw_logger.cc'] + src += ['encoder/ffmpeg_encoder.cc'] if arch == "Darwin": # fix OpenCL @@ -20,8 +20,7 @@ logger_lib = env.Library('logger', src) libs.insert(0, logger_lib) env.Program('loggerd', ['main.cc'], LIBS=libs) -if arch == "larch64": - env.Program('encoderd', ['encoderd.cc'], LIBS=libs) +env.Program('encoderd', ['encoderd.cc'], LIBS=libs) env.Program('bootlog.cc', LIBS=libs) if GetOption('test'): diff --git a/selfdrive/loggerd/encoder.h b/selfdrive/loggerd/encoder.h deleted file mode 100644 index 0a5afa007c..0000000000 --- a/selfdrive/loggerd/encoder.h +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -#include -#include "cereal/visionipc/visionipc.h" - -class VideoEncoder { -public: - virtual ~VideoEncoder() {} - virtual int encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const uint8_t *v_ptr, - int in_width, int in_height, VisionIpcBufExtra *extra) = 0; - virtual void encoder_open(const char* path) = 0; - virtual void encoder_close() = 0; -}; diff --git a/selfdrive/loggerd/encoder/encoder.cc b/selfdrive/loggerd/encoder/encoder.cc new file mode 100644 index 0000000000..84f26d9787 --- /dev/null +++ b/selfdrive/loggerd/encoder/encoder.cc @@ -0,0 +1,79 @@ +#include +#include "selfdrive/loggerd/encoder/encoder.h" + +VideoEncoder::~VideoEncoder() {} + +void VideoEncoder::publisher_init() { + // publish + service_name = this->type == DriverCam ? "driverEncodeData" : + (this->type == WideRoadCam ? "wideRoadEncodeData" : + (this->in_width == this->out_width ? "roadEncodeData" : "qRoadEncodeData")); + pm.reset(new PubMaster({service_name})); +} + +void VideoEncoder::publisher_publish(VideoEncoder *e, int segment_num, uint32_t idx, VisionIpcBufExtra &extra, + unsigned int flags, kj::ArrayPtr header, kj::ArrayPtr dat) { + // broadcast packet + MessageBuilder msg; + auto event = msg.initEvent(true); + auto edat = (e->type == DriverCam) ? event.initDriverEncodeData() : + ((e->type == WideRoadCam) ? event.initWideRoadEncodeData() : + (e->in_width == e->out_width ? event.initRoadEncodeData() : event.initQRoadEncodeData())); + auto edata = edat.initIdx(); + edata.setFrameId(extra.frame_id); + edata.setTimestampSof(extra.timestamp_sof); + edata.setTimestampEof(extra.timestamp_eof); + edata.setType(e->codec); + edata.setEncodeId(idx); + edata.setSegmentNum(segment_num); + edata.setSegmentId(idx); + edata.setFlags(flags); + edata.setLen(dat.size()); + edat.setData(dat); + if (flags & V4L2_BUF_FLAG_KEYFRAME) edat.setHeader(header); + + auto words = new kj::Array(capnp::messageToFlatArray(msg)); + auto bytes = words->asBytes(); + e->pm->send(e->service_name, bytes.begin(), bytes.size()); + if (e->write) { + e->to_write.push(words); + } else { + delete words; + } +} + +// TODO: writing should be moved to loggerd +void VideoEncoder::write_handler(VideoEncoder *e, const char *path) { + VideoWriter writer(path, e->filename, e->codec != cereal::EncodeIndex::Type::FULL_H_E_V_C, e->out_width, e->out_height, e->fps, e->codec); + + bool first = true; + kj::Array* out_buf; + while ((out_buf = e->to_write.pop())) { + capnp::FlatArrayMessageReader cmsg(*out_buf); + cereal::Event::Reader event = cmsg.getRoot(); + + auto edata = (e->type == DriverCam) ? event.getDriverEncodeData() : + ((e->type == WideRoadCam) ? event.getWideRoadEncodeData() : + (e->in_width == e->out_width ? event.getRoadEncodeData() : event.getQRoadEncodeData())); + auto idx = edata.getIdx(); + auto flags = idx.getFlags(); + + if (first) { + assert(flags & V4L2_BUF_FLAG_KEYFRAME); + auto header = edata.getHeader(); + writer.write((uint8_t *)header.begin(), header.size(), idx.getTimestampEof()/1000, true, false); + first = false; + } + + // dangerous cast from const, but should be fine + auto data = edata.getData(); + if (data.size() > 0) { + writer.write((uint8_t *)data.begin(), data.size(), idx.getTimestampEof()/1000, false, flags & V4L2_BUF_FLAG_KEYFRAME); + } + + // free the data + delete out_buf; + } + + // VideoWriter is freed on out of scope +} diff --git a/selfdrive/loggerd/encoder/encoder.h b/selfdrive/loggerd/encoder/encoder.h new file mode 100644 index 0000000000..abdcc9c4bf --- /dev/null +++ b/selfdrive/loggerd/encoder/encoder.h @@ -0,0 +1,60 @@ +#pragma once + +#include +#include +#include + +#include "cereal/messaging/messaging.h" +#include "cereal/visionipc/visionipc.h" +#include "selfdrive/common/queue.h" +#include "selfdrive/loggerd/encoder/video_writer.h" +#include "selfdrive/camerad/cameras/camera_common.h" + +#define V4L2_BUF_FLAG_KEYFRAME 8 + +class VideoEncoder { +public: + VideoEncoder(const char* filename, CameraType type, int in_width, int in_height, int fps, + int bitrate, cereal::EncodeIndex::Type codec, int out_width, int out_height, bool write) + : filename(filename), type(type), in_width(in_width), in_height(in_height), fps(fps), + bitrate(bitrate), codec(codec), out_width(out_width), out_height(out_height), write(write) { } + virtual ~VideoEncoder(); + virtual int encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const uint8_t *v_ptr, + int in_width, int in_height, VisionIpcBufExtra *extra) = 0; + virtual void encoder_open(const char* path) = 0; + virtual void encoder_close() = 0; + + void publisher_init(); + static void publisher_publish(VideoEncoder *e, int segment_num, uint32_t idx, VisionIpcBufExtra &extra, unsigned int flags, kj::ArrayPtr header, kj::ArrayPtr dat); + + void writer_open(const char* path) { + if (this->write) write_handler_thread = std::thread(VideoEncoder::write_handler, this, path); + } + + void writer_close() { + if (this->write) { + to_write.push(NULL); + write_handler_thread.join(); + } + assert(to_write.empty()); + } + +protected: + bool write; + const char* filename; + int in_width, in_height; + int out_width, out_height, fps; + int bitrate; + cereal::EncodeIndex::Type codec; + CameraType type; + +private: + // publishing + std::unique_ptr pm; + const char *service_name; + + // writing support + static void write_handler(VideoEncoder *e, const char *path); + std::thread write_handler_thread; + SafeQueue* > to_write; +}; diff --git a/selfdrive/loggerd/raw_logger.cc b/selfdrive/loggerd/encoder/ffmpeg_encoder.cc similarity index 58% rename from selfdrive/loggerd/raw_logger.cc rename to selfdrive/loggerd/encoder/ffmpeg_encoder.cc index 72c8d0d93e..139ffe3f24 100644 --- a/selfdrive/loggerd/raw_logger.cc +++ b/selfdrive/loggerd/encoder/ffmpeg_encoder.cc @@ -1,6 +1,6 @@ #pragma clang diagnostic ignored "-Wdeprecated-declarations" -#include "selfdrive/loggerd/raw_logger.h" +#include "selfdrive/loggerd/encoder/ffmpeg_encoder.h" #include #include @@ -22,10 +22,9 @@ extern "C" { #include "selfdrive/common/swaglog.h" #include "selfdrive/common/util.h" -RawLogger::RawLogger(const char* filename, CameraType type, int in_width, int in_height, int fps, - int bitrate, bool h265, int out_width, int out_height, bool write) - : in_width_(in_width), in_height_(in_height), filename(filename), fps(fps) { - // TODO: respect write arg +const int env_debug_encoder = (getenv("DEBUG_ENCODER") != NULL) ? atoi(getenv("DEBUG_ENCODER")) : 0; + +void FfmpegEncoder::encoder_init() { frame = av_frame_alloc(); assert(frame); frame->format = AV_PIX_FMT_YUV420P; @@ -38,30 +37,43 @@ RawLogger::RawLogger(const char* filename, CameraType type, int in_width, int in if (in_width != out_width || in_height != out_height) { downscale_buf.resize(out_width * out_height * 3 / 2); } + + publisher_init(); } -RawLogger::~RawLogger() { +FfmpegEncoder::~FfmpegEncoder() { encoder_close(); av_frame_free(&frame); } -void RawLogger::encoder_open(const char* path) { - writer = new VideoWriter(path, this->filename, true, frame->width, frame->height, this->fps, false, true); - // write the header - writer->write(NULL, 0, 0, true, false); +void FfmpegEncoder::encoder_open(const char* path) { + AVCodec *codec = avcodec_find_encoder(AV_CODEC_ID_FFVHUFF); + + this->codec_ctx = avcodec_alloc_context3(codec); + assert(this->codec_ctx); + this->codec_ctx->width = frame->width; + this->codec_ctx->height = frame->height; + this->codec_ctx->pix_fmt = AV_PIX_FMT_YUV420P; + this->codec_ctx->time_base = (AVRational){ 1, fps }; + int err = avcodec_open2(this->codec_ctx, codec, NULL); + assert(err >= 0); + + writer_open(path); is_open = true; + segment_num++; + counter = 0; } -void RawLogger::encoder_close() { +void FfmpegEncoder::encoder_close() { if (!is_open) return; - delete writer; + writer_close(); is_open = false; } -int RawLogger::encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const uint8_t *v_ptr, - int in_width, int in_height, VisionIpcBufExtra *extra) { - assert(in_width == this->in_width_); - assert(in_height == this->in_height_); +int FfmpegEncoder::encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const uint8_t *v_ptr, + int in_width_, int in_height_, VisionIpcBufExtra *extra) { + assert(in_width_ == this->in_width); + assert(in_height_ == this->in_height); if (downscale_buf.size() > 0) { uint8_t *out_y = downscale_buf.data(); @@ -88,7 +100,7 @@ int RawLogger::encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const ui int ret = counter; - int err = avcodec_send_frame(writer->codec_ctx, frame); + int err = avcodec_send_frame(this->codec_ctx, frame); if (err < 0) { LOGE("avcodec_send_frame error %d", err); ret = -1; @@ -99,7 +111,7 @@ int RawLogger::encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const ui pkt.data = NULL; pkt.size = 0; while (ret >= 0) { - err = avcodec_receive_packet(writer->codec_ctx, &pkt); + err = avcodec_receive_packet(this->codec_ctx, &pkt); if (err == AVERROR_EOF) { break; } else if (err == AVERROR(EAGAIN)) { @@ -112,7 +124,15 @@ int RawLogger::encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const ui break; } - writer->write(pkt.data, pkt.size, pkt.pts, false, pkt.flags & AV_PKT_FLAG_KEY); + if (env_debug_encoder) { + printf("%20s got %8d bytes flags %8x idx %4d id %8d\n", this->filename, pkt.size, pkt.flags, counter, extra->frame_id); + } + + publisher_publish(this, segment_num, counter, *extra, + (pkt.flags & AV_PKT_FLAG_KEY) ? V4L2_BUF_FLAG_KEYFRAME : 0, + kj::arrayPtr(pkt.data, (size_t)0), // TODO: get the header + kj::arrayPtr(pkt.data, pkt.size)); + counter++; } av_packet_unref(&pkt); diff --git a/selfdrive/loggerd/encoder/ffmpeg_encoder.h b/selfdrive/loggerd/encoder/ffmpeg_encoder.h new file mode 100644 index 0000000000..5f90e198fc --- /dev/null +++ b/selfdrive/loggerd/encoder/ffmpeg_encoder.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include +#include + +extern "C" { +#include +#include +#include +} + +#include "selfdrive/loggerd/encoder/encoder.h" +#include "selfdrive/loggerd/loggerd.h" +#include "selfdrive/loggerd/encoder/video_writer.h" + +class FfmpegEncoder : public VideoEncoder { + public: + FfmpegEncoder(const char* filename, CameraType type, int in_width, int in_height, int fps, + int bitrate, cereal::EncodeIndex::Type codec, int out_width, int out_height, bool write) : + VideoEncoder(filename, type, in_width, in_height, fps, bitrate, cereal::EncodeIndex::Type::BIG_BOX_LOSSLESS, out_width, out_height, write) { encoder_init(); } + ~FfmpegEncoder(); + void encoder_init(); + int encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const uint8_t *v_ptr, + int in_width_, int in_height_, VisionIpcBufExtra *extra); + void encoder_open(const char* path); + void encoder_close(); + +private: + int segment_num = -1; + int counter = 0; + bool is_open = false; + + AVCodecContext *codec_ctx; + AVFrame *frame = NULL; + std::vector downscale_buf; +}; diff --git a/selfdrive/loggerd/v4l_encoder.cc b/selfdrive/loggerd/encoder/v4l_encoder.cc similarity index 75% rename from selfdrive/loggerd/v4l_encoder.cc rename to selfdrive/loggerd/encoder/v4l_encoder.cc index 99653758a0..f5608a067e 100644 --- a/selfdrive/loggerd/v4l_encoder.cc +++ b/selfdrive/loggerd/encoder/v4l_encoder.cc @@ -2,7 +2,7 @@ #include #include -#include "selfdrive/loggerd/v4l_encoder.h" +#include "selfdrive/loggerd/encoder/v4l_encoder.h" #include "selfdrive/common/util.h" #include "selfdrive/common/timing.h" @@ -67,42 +67,6 @@ static void request_buffers(int fd, v4l2_buf_type buf_type, unsigned int count) checked_ioctl(fd, VIDIOC_REQBUFS, &reqbuf); } -// TODO: writing should be moved to loggerd -void V4LEncoder::write_handler(V4LEncoder *e, const char *path) { - VideoWriter writer(path, e->filename, !e->h265, e->width, e->height, e->fps, e->h265, false); - - bool first = true; - kj::Array* out_buf; - while ((out_buf = e->to_write.pop())) { - capnp::FlatArrayMessageReader cmsg(*out_buf); - cereal::Event::Reader event = cmsg.getRoot(); - - auto edata = (e->type == DriverCam) ? event.getDriverEncodeData() : - ((e->type == WideRoadCam) ? event.getWideRoadEncodeData() : - (e->h265 ? event.getRoadEncodeData() : event.getQRoadEncodeData())); - auto idx = edata.getIdx(); - auto flags = idx.getFlags(); - - if (first) { - assert(flags & V4L2_BUF_FLAG_KEYFRAME); - auto header = edata.getHeader(); - writer.write((uint8_t *)header.begin(), header.size(), idx.getTimestampEof()/1000, true, false); - first = false; - } - - // dangerous cast from const, but should be fine - auto data = edata.getData(); - if (data.size() > 0) { - writer.write((uint8_t *)data.begin(), data.size(), idx.getTimestampEof()/1000, false, flags & V4L2_BUF_FLAG_KEYFRAME); - } - - // free the data - delete out_buf; - } - - // VideoWriter is freed on out of scope -} - void V4LEncoder::dequeue_handler(V4LEncoder *e) { std::string dequeue_thread_name = "dq-"+std::string(e->filename); util::set_thread_name(dequeue_thread_name.c_str()); @@ -138,7 +102,6 @@ void V4LEncoder::dequeue_handler(V4LEncoder *e) { // eof packet, we exit if (flags & V4L2_QCOM_BUF_FLAG_EOS) { - if (e->write) e->to_write.push(NULL); exit = true; } else if (flags & V4L2_QCOM_BUF_FLAG_CODECCONFIG) { // save header @@ -146,37 +109,9 @@ void V4LEncoder::dequeue_handler(V4LEncoder *e) { } else { VisionIpcBufExtra extra = e->extras.pop(); assert(extra.timestamp_eof/1000 == ts); // stay in sync - frame_id = extra.frame_id; ++idx; - - // broadcast packet - MessageBuilder msg; - auto event = msg.initEvent(true); - auto edat = (e->type == DriverCam) ? event.initDriverEncodeData() : - ((e->type == WideRoadCam) ? event.initWideRoadEncodeData() : - (e->h265 ? event.initRoadEncodeData() : event.initQRoadEncodeData())); - auto edata = edat.initIdx(); - edata.setFrameId(extra.frame_id); - edata.setTimestampSof(extra.timestamp_sof); - edata.setTimestampEof(extra.timestamp_eof); - edata.setType(e->h265 ? cereal::EncodeIndex::Type::FULL_H_E_V_C : cereal::EncodeIndex::Type::QCAMERA_H264); - edata.setEncodeId(idx); - edata.setSegmentNum(e->segment_num); - edata.setSegmentId(idx); - edata.setFlags(flags); - edata.setLen(bytesused); - edat.setData(kj::arrayPtr(buf, bytesused)); - if (flags & V4L2_BUF_FLAG_KEYFRAME) edat.setHeader(header); - - auto words = new kj::Array(capnp::messageToFlatArray(msg)); - auto bytes = words->asBytes(); - e->pm->send(e->service_name, bytes.begin(), bytes.size()); - if (e->write) { - e->to_write.push(words); - } else { - delete words; - } + e->publisher_publish(e, e->segment_num, idx, extra, flags, header, kj::arrayPtr(buf, bytesused)); } if (env_debug_encoder) { @@ -196,12 +131,7 @@ void V4LEncoder::dequeue_handler(V4LEncoder *e) { } } -V4LEncoder::V4LEncoder( - const char* filename, CameraType type, int in_width, int in_height, - int fps, int bitrate, bool h265, int out_width, int out_height, bool write) - : type(type), in_width_(in_width), in_height_(in_height), - filename(filename), h265(h265), - width(out_width), height(out_height), fps(fps), write(write) { +void V4LEncoder::encoder_init() { fd = open("/dev/v4l/by-path/platform-aa00000.qcom_vidc-video-index1", O_RDWR|O_NONBLOCK); assert(fd >= 0); @@ -218,7 +148,7 @@ V4LEncoder::V4LEncoder( // downscales are free with v4l .width = (unsigned int)out_width, .height = (unsigned int)out_height, - .pixelformat = h265 ? V4L2_PIX_FMT_HEVC : V4L2_PIX_FMT_H264, + .pixelformat = (codec == cereal::EncodeIndex::Type::FULL_H_E_V_C) ? V4L2_PIX_FMT_HEVC : V4L2_PIX_FMT_H264, .field = V4L2_FIELD_ANY, .colorspace = V4L2_COLORSPACE_DEFAULT, } @@ -272,7 +202,7 @@ V4LEncoder::V4LEncoder( } } - if (h265) { + if (codec == cereal::EncodeIndex::Type::FULL_H_E_V_C) { struct v4l2_control ctrls[] = { { .id = V4L2_CID_MPEG_VIDC_VIDEO_HEVC_PROFILE, .value = V4L2_MPEG_VIDC_VIDEO_HEVC_PROFILE_MAIN}, { .id = V4L2_CID_MPEG_VIDC_VIDEO_HEVC_TIER_LEVEL, .value = V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_5}, @@ -321,23 +251,18 @@ V4LEncoder::V4LEncoder( free_buf_in.push(i); } - // publish - service_name = this->type == DriverCam ? "driverEncodeData" : - (this->type == WideRoadCam ? "wideRoadEncodeData" : - (this->h265 ? "roadEncodeData" : "qRoadEncodeData")); - pm.reset(new PubMaster({service_name})); + publisher_init(); } - void V4LEncoder::encoder_open(const char* path) { dequeue_handler_thread = std::thread(V4LEncoder::dequeue_handler, this); - if (this->write) write_handler_thread = std::thread(V4LEncoder::write_handler, this, path); + writer_open(path); this->is_open = true; this->counter = 0; } int V4LEncoder::encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const uint8_t *v_ptr, - int in_width, int in_height, VisionIpcBufExtra *extra) { + int in_width_, int in_height_, VisionIpcBufExtra *extra) { assert(in_width == in_width_); assert(in_height == in_height_); assert(is_open); @@ -383,8 +308,7 @@ void V4LEncoder::encoder_close() { // join waits for V4L2_QCOM_BUF_FLAG_EOS dequeue_handler_thread.join(); assert(extras.empty()); - if (this->write) write_handler_thread.join(); - assert(to_write.empty()); + writer_close(); } this->is_open = false; } diff --git a/selfdrive/loggerd/v4l_encoder.h b/selfdrive/loggerd/encoder/v4l_encoder.h similarity index 52% rename from selfdrive/loggerd/v4l_encoder.h rename to selfdrive/loggerd/encoder/v4l_encoder.h index 43b70baebd..13ebab20da 100644 --- a/selfdrive/loggerd/v4l_encoder.h +++ b/selfdrive/loggerd/encoder/v4l_encoder.h @@ -1,17 +1,19 @@ #pragma once #include "selfdrive/common/queue.h" -#include "selfdrive/loggerd/encoder.h" -#include "selfdrive/loggerd/loggerd.h" -#include "selfdrive/loggerd/video_writer.h" +#include "selfdrive/loggerd/encoder/encoder.h" +#include "selfdrive/loggerd/encoder/video_writer.h" #define BUF_IN_COUNT 7 #define BUF_OUT_COUNT 6 class V4LEncoder : public VideoEncoder { public: - V4LEncoder(const char* filename, CameraType type, int width, int height, int fps, int bitrate, bool h265, int out_width, int out_height, bool write = true); + V4LEncoder(const char* filename, CameraType type, int in_width, int in_height, int fps, + int bitrate, cereal::EncodeIndex::Type codec, int out_width, int out_height, bool write) : + VideoEncoder(filename, type, in_width, in_height, fps, bitrate, codec, out_width, out_height, write) { encoder_init(); } ~V4LEncoder(); + void encoder_init(); int encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const uint8_t *v_ptr, int in_width, int in_height, VisionIpcBufExtra *extra); void encoder_open(const char* path); @@ -19,16 +21,11 @@ public: private: int fd; - const char* filename; - CameraType type; - unsigned int in_width_, in_height_; - bool h265; bool is_open = false; int segment_num = -1; int counter = 0; - std::unique_ptr pm; - const char *service_name; + SafeQueue extras; static void dequeue_handler(V4LEncoder *e); std::thread dequeue_handler_thread; @@ -36,13 +33,4 @@ private: VisionBuf buf_in[BUF_IN_COUNT]; VisionBuf buf_out[BUF_OUT_COUNT]; SafeQueue free_buf_in; - - SafeQueue extras; - - // writing support - int width, height, fps; - bool write; - static void write_handler(V4LEncoder *e, const char *path); - std::thread write_handler_thread; - SafeQueue* > to_write; }; diff --git a/selfdrive/loggerd/video_writer.cc b/selfdrive/loggerd/encoder/video_writer.cc similarity index 83% rename from selfdrive/loggerd/video_writer.cc rename to selfdrive/loggerd/encoder/video_writer.cc index 51e606dc5c..c02592583a 100644 --- a/selfdrive/loggerd/video_writer.cc +++ b/selfdrive/loggerd/encoder/video_writer.cc @@ -3,12 +3,13 @@ #include #include -#include "selfdrive/loggerd/video_writer.h" +#include "selfdrive/loggerd/encoder/video_writer.h" #include "selfdrive/common/swaglog.h" #include "selfdrive/common/util.h" -VideoWriter::VideoWriter(const char *path, const char *filename, bool remuxing, int width, int height, int fps, bool h265, bool raw) - : remuxing(remuxing), raw(raw) { +VideoWriter::VideoWriter(const char *path, const char *filename, bool remuxing, int width, int height, int fps, cereal::EncodeIndex::Type codec) + : remuxing(remuxing) { + raw = codec == cereal::EncodeIndex::Type::BIG_BOX_LOSSLESS; vid_path = util::string_format("%s/%s", path, filename); lock_path = util::string_format("%s/%s.lock", path, filename); @@ -24,25 +25,25 @@ VideoWriter::VideoWriter(const char *path, const char *filename, bool remuxing, // set codec correctly. needed? av_register_all(); - AVCodec *codec = NULL; - assert(!h265); - codec = avcodec_find_encoder(raw ? AV_CODEC_ID_FFVHUFF : AV_CODEC_ID_H264); - assert(codec); + AVCodec *avcodec = NULL; + assert(codec != cereal::EncodeIndex::Type::FULL_H_E_V_C); + avcodec = avcodec_find_encoder(raw ? AV_CODEC_ID_FFVHUFF : AV_CODEC_ID_H264); + assert(avcodec); - this->codec_ctx = avcodec_alloc_context3(codec); + this->codec_ctx = avcodec_alloc_context3(avcodec); assert(this->codec_ctx); this->codec_ctx->width = width; this->codec_ctx->height = height; this->codec_ctx->pix_fmt = AV_PIX_FMT_YUV420P; this->codec_ctx->time_base = (AVRational){ 1, fps }; - if (raw) { - // since the codec is actually used, we open it - int err = avcodec_open2(this->codec_ctx, codec, NULL); + if (codec == cereal::EncodeIndex::Type::BIG_BOX_LOSSLESS) { + // without this, there's just noise + int err = avcodec_open2(this->codec_ctx, avcodec, NULL); assert(err >= 0); } - this->out_stream = avformat_new_stream(this->ofmt_ctx, raw ? codec : NULL); + this->out_stream = avformat_new_stream(this->ofmt_ctx, raw ? avcodec : NULL); assert(this->out_stream); int err = avio_open(&this->ofmt_ctx->pb, this->vid_path.c_str(), AVIO_FLAG_WRITE); @@ -64,7 +65,7 @@ void VideoWriter::write(uint8_t *data, int len, long long timestamp, bool codecc if (remuxing) { if (codecconfig) { - if (data) { + if (len > 0) { codec_ctx->extradata = (uint8_t*)av_mallocz(len + AV_INPUT_BUFFER_PADDING_SIZE); codec_ctx->extradata_size = len; memcpy(codec_ctx->extradata, data, len); diff --git a/selfdrive/loggerd/video_writer.h b/selfdrive/loggerd/encoder/video_writer.h similarity index 80% rename from selfdrive/loggerd/video_writer.h rename to selfdrive/loggerd/encoder/video_writer.h index 8048d93a07..217964e0ef 100644 --- a/selfdrive/loggerd/video_writer.h +++ b/selfdrive/loggerd/encoder/video_writer.h @@ -6,17 +6,19 @@ extern "C" { #include } +#include "cereal/messaging/messaging.h" + class VideoWriter { public: - VideoWriter(const char *path, const char *filename, bool remuxing, int width, int height, int fps, bool h265, bool raw); + VideoWriter(const char *path, const char *filename, bool remuxing, int width, int height, int fps, cereal::EncodeIndex::Type codec); void write(uint8_t *data, int len, long long timestamp, bool codecconfig, bool keyframe); ~VideoWriter(); - AVCodecContext *codec_ctx; private: std::string vid_path, lock_path; FILE *of = nullptr; + AVCodecContext *codec_ctx; AVFormatContext *ofmt_ctx; AVStream *out_stream; bool remuxing, raw; diff --git a/selfdrive/loggerd/encoderd.cc b/selfdrive/loggerd/encoderd.cc index ac57d3eea5..18e88ca8bb 100644 --- a/selfdrive/loggerd/encoderd.cc +++ b/selfdrive/loggerd/encoderd.cc @@ -54,12 +54,14 @@ void encoder_thread(EncoderdState *s, const LogCameraInfo &cam_info) { // main encoder encoders.push_back(new Encoder(cam_info.filename, cam_info.type, buf_info.width, buf_info.height, - cam_info.fps, cam_info.bitrate, cam_info.is_h265, + cam_info.fps, cam_info.bitrate, + cam_info.is_h265 ? cereal::EncodeIndex::Type::FULL_H_E_V_C : cereal::EncodeIndex::Type::QCAMERA_H264, buf_info.width, buf_info.height, false)); // qcamera encoder if (cam_info.has_qcamera) { encoders.push_back(new Encoder(qcam_info.filename, cam_info.type, buf_info.width, buf_info.height, - qcam_info.fps, qcam_info.bitrate, qcam_info.is_h265, + qcam_info.fps, qcam_info.bitrate, + qcam_info.is_h265 ? cereal::EncodeIndex::Type::FULL_H_E_V_C : cereal::EncodeIndex::Type::QCAMERA_H264, qcam_info.frame_width, qcam_info.frame_height, false)); } } diff --git a/selfdrive/loggerd/loggerd.cc b/selfdrive/loggerd/loggerd.cc index 8e488b4a93..5eb7e6b8cd 100644 --- a/selfdrive/loggerd/loggerd.cc +++ b/selfdrive/loggerd/loggerd.cc @@ -65,13 +65,15 @@ void encoder_thread(LoggerdState *s, const LogCameraInfo &cam_info) { // main encoder encoders.push_back(new Encoder(cam_info.filename, cam_info.type, buf_info.width, buf_info.height, - cam_info.fps, cam_info.bitrate, cam_info.is_h265, + cam_info.fps, cam_info.bitrate, + cam_info.is_h265 ? cereal::EncodeIndex::Type::FULL_H_E_V_C : cereal::EncodeIndex::Type::QCAMERA_H264, buf_info.width, buf_info.height, cam_info.record)); // qcamera encoder if (cam_info.has_qcamera) { encoders.push_back(new Encoder(qcam_info.filename, cam_info.type, buf_info.width, buf_info.height, - qcam_info.fps, qcam_info.bitrate, qcam_info.is_h265, - qcam_info.frame_width, qcam_info.frame_height)); + qcam_info.fps, qcam_info.bitrate, + qcam_info.is_h265 ? cereal::EncodeIndex::Type::FULL_H_E_V_C : cereal::EncodeIndex::Type::QCAMERA_H264, + qcam_info.frame_width, qcam_info.frame_height, true)); } } diff --git a/selfdrive/loggerd/loggerd.h b/selfdrive/loggerd/loggerd.h index ec014649a8..447c4fb4c9 100644 --- a/selfdrive/loggerd/loggerd.h +++ b/selfdrive/loggerd/loggerd.h @@ -22,14 +22,14 @@ #include "selfdrive/common/util.h" #include "selfdrive/hardware/hw.h" -#include "selfdrive/loggerd/encoder.h" +#include "selfdrive/loggerd/encoder/encoder.h" #include "selfdrive/loggerd/logger.h" #ifdef QCOM2 -#include "selfdrive/loggerd/v4l_encoder.h" +#include "selfdrive/loggerd/encoder/v4l_encoder.h" #define Encoder V4LEncoder #else -#include "selfdrive/loggerd/raw_logger.h" -#define Encoder RawLogger +#include "selfdrive/loggerd/encoder/ffmpeg_encoder.h" +#define Encoder FfmpegEncoder #endif constexpr int MAIN_FPS = 20; diff --git a/selfdrive/loggerd/raw_logger.h b/selfdrive/loggerd/raw_logger.h deleted file mode 100644 index 14c4a61ea0..0000000000 --- a/selfdrive/loggerd/raw_logger.h +++ /dev/null @@ -1,41 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -extern "C" { -#include -#include -#include -} - -#include "selfdrive/loggerd/encoder.h" -#include "selfdrive/loggerd/loggerd.h" -#include "selfdrive/loggerd/video_writer.h" - -class RawLogger : public VideoEncoder { - public: - RawLogger(const char* filename, CameraType type, int in_width, int in_height, int fps, - int bitrate, bool h265, int out_width, int out_height, bool write = true); - ~RawLogger(); - int encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const uint8_t *v_ptr, - int in_width, int in_height, VisionIpcBufExtra *extra); - void encoder_open(const char* path); - void encoder_close(); - -private: - const char* filename; - //bool write; - int fps; - int counter = 0; - bool is_open = false; - - int in_width_, in_height_; - - AVFrame *frame = NULL; - std::vector downscale_buf; - - VideoWriter *writer = NULL; -}; diff --git a/selfdrive/loggerd/remote_encoder.cc b/selfdrive/loggerd/remote_encoder.cc index c79595b0fe..964cd561f0 100644 --- a/selfdrive/loggerd/remote_encoder.cc +++ b/selfdrive/loggerd/remote_encoder.cc @@ -40,7 +40,7 @@ int handle_encoder_msg(LoggerdState *s, Message *msg, std::string &name, struct re.writer.reset(new VideoWriter(s->segment_path, cam_info.filename, !cam_info.is_h265, cam_info.frame_width, cam_info.frame_height, - cam_info.fps, cam_info.is_h265, false)); + cam_info.fps, cam_info.is_h265 ? cereal::EncodeIndex::Type::FULL_H_E_V_C : cereal::EncodeIndex::Type::QCAMERA_H264)); // write the header auto header = edata.getHeader(); re.writer->write((uint8_t *)header.begin(), header.size(), idx.getTimestampEof()/1000, true, false); diff --git a/selfdrive/loggerd/remote_encoder.h b/selfdrive/loggerd/remote_encoder.h index 6fc91a2209..79e628f514 100644 --- a/selfdrive/loggerd/remote_encoder.h +++ b/selfdrive/loggerd/remote_encoder.h @@ -1,5 +1,4 @@ -#include "selfdrive/loggerd/video_writer.h" -#define V4L2_BUF_FLAG_KEYFRAME 0x00000008 +#include "selfdrive/loggerd/encoder/video_writer.h" struct RemoteEncoder { std::unique_ptr writer; diff --git a/tools/camerastream/compressed_vipc.py b/tools/camerastream/compressed_vipc.py index c39ac955e6..7b6f755ea7 100755 --- a/tools/camerastream/compressed_vipc.py +++ b/tools/camerastream/compressed_vipc.py @@ -71,7 +71,7 @@ def decoder(addr, sock_name, vipc_server, vst, nvidia): assert len(frames) == 1 img_yuv = frames[0].to_ndarray(format=av.video.format.VideoFormat('yuv420p')) - vipc_server.send(vst, img_yuv.flatten().data, cnt, 0, 0) + vipc_server.send(vst, img_yuv.flatten().data, cnt, cnt*1e9/20, cnt*1e9/20) cnt += 1 pc_latency = (time.monotonic()-time_q[0])*1000