encoderd: make work on PC (#24483)

* don't use the codec in video_writer

* this produces broken videos for some reason

* bugfix

* refactor on the class

* works on device

* fix codec

* no codec enum

* fix pc

* move into dirs

* these includes also

* rename it ffmpegencoder

* add avcodec_close

Co-authored-by: Comma Device <device@comma.ai>
old-commit-hash: 2008db47c2

cereal

@@ -1 +1 @@
-Subproject commit c7d3a0acbae267ef93d30044e1941e060dac9e48
+Subproject commit 67e5c5ca378fcf6db53e968e5ad9e4a98d32ed0f

release/files_common

@@ -298,11 +298,12 @@ selfdrive/proclogd/proclog.cc
 selfdrive/proclogd/proclog.h
 
 selfdrive/loggerd/SConscript
-selfdrive/loggerd/encoder.h
-selfdrive/loggerd/v4l_encoder.cc
-selfdrive/loggerd/v4l_encoder.h
-selfdrive/loggerd/video_writer.cc
-selfdrive/loggerd/video_writer.h
+selfdrive/loggerd/encoder/encoder.cc
+selfdrive/loggerd/encoder/encoder.h
+selfdrive/loggerd/encoder/v4l_encoder.cc
+selfdrive/loggerd/encoder/v4l_encoder.h
+selfdrive/loggerd/encoder/video_writer.cc
+selfdrive/loggerd/encoder/video_writer.h
 selfdrive/loggerd/remote_encoder.cc
 selfdrive/loggerd/remote_encoder.h
 selfdrive/loggerd/logger.cc
@@ -312,8 +313,8 @@ selfdrive/loggerd/loggerd.h
 selfdrive/loggerd/encoderd.cc
 selfdrive/loggerd/main.cc
 selfdrive/loggerd/bootlog.cc
-selfdrive/loggerd/raw_logger.cc
-selfdrive/loggerd/raw_logger.h
+selfdrive/loggerd/encoder/ffmpeg_encoder.cc
+selfdrive/loggerd/encoder/ffmpeg_encoder.h
 
 selfdrive/loggerd/__init__.py
 selfdrive/loggerd/config.py

selfdrive/loggerd/SConscript

@@ -5,11 +5,11 @@ libs = [common, cereal, messaging, visionipc,
         'avformat', 'avcodec', 'swscale', 'avutil',
         'yuv', 'OpenCL', 'pthread']
 
-src = ['logger.cc', 'loggerd.cc', 'video_writer.cc', 'remote_encoder.cc']
+src = ['logger.cc', 'loggerd.cc', 'encoder/video_writer.cc', 'remote_encoder.cc', 'encoder/encoder.cc']
 if arch == "larch64":
-  src += ['v4l_encoder.cc']
+  src += ['encoder/v4l_encoder.cc']
 else:
-  src += ['raw_logger.cc']
+  src += ['encoder/ffmpeg_encoder.cc']
 
 if arch == "Darwin":
   # fix OpenCL
@@ -20,7 +20,6 @@ logger_lib = env.Library('logger', src)
 libs.insert(0, logger_lib)
 
 env.Program('loggerd', ['main.cc'], LIBS=libs)
-if arch == "larch64":
 env.Program('encoderd', ['encoderd.cc'], LIBS=libs)
 env.Program('bootlog.cc', LIBS=libs)
 

selfdrive/loggerd/encoder.h

@@ -1,13 +0,0 @@
-#pragma once
-
-#include <cstdint>
-#include "cereal/visionipc/visionipc.h"
-
-class VideoEncoder {
-public:
-  virtual ~VideoEncoder() {}
-  virtual int encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const uint8_t *v_ptr,
-                           int in_width, int in_height, VisionIpcBufExtra *extra) = 0;
-  virtual void encoder_open(const char* path) = 0;
-  virtual void encoder_close() = 0;
-};

selfdrive/loggerd/encoder/encoder.cc

@@ -0,0 +1,79 @@
+#include <cassert>
+#include "selfdrive/loggerd/encoder/encoder.h"
+
+VideoEncoder::~VideoEncoder() {}
+
+void VideoEncoder::publisher_init() {
+  // publish
+  service_name = this->type == DriverCam ? "driverEncodeData" :
+    (this->type == WideRoadCam ? "wideRoadEncodeData" :
+    (this->in_width == this->out_width ? "roadEncodeData" : "qRoadEncodeData"));
+  pm.reset(new PubMaster({service_name}));
+}
+
+void VideoEncoder::publisher_publish(VideoEncoder *e, int segment_num, uint32_t idx, VisionIpcBufExtra &extra,
+                                     unsigned int flags, kj::ArrayPtr<capnp::byte> header, kj::ArrayPtr<capnp::byte> dat) {
+  // broadcast packet
+  MessageBuilder msg;
+  auto event = msg.initEvent(true);
+  auto edat = (e->type == DriverCam) ? event.initDriverEncodeData() :
+    ((e->type == WideRoadCam) ? event.initWideRoadEncodeData() :
+    (e->in_width == e->out_width ? event.initRoadEncodeData() : event.initQRoadEncodeData()));
+  auto edata = edat.initIdx();
+  edata.setFrameId(extra.frame_id);
+  edata.setTimestampSof(extra.timestamp_sof);
+  edata.setTimestampEof(extra.timestamp_eof);
+  edata.setType(e->codec);
+  edata.setEncodeId(idx);
+  edata.setSegmentNum(segment_num);
+  edata.setSegmentId(idx);
+  edata.setFlags(flags);
+  edata.setLen(dat.size());
+  edat.setData(dat);
+  if (flags & V4L2_BUF_FLAG_KEYFRAME) edat.setHeader(header);
+
+  auto words = new kj::Array<capnp::word>(capnp::messageToFlatArray(msg));
+  auto bytes = words->asBytes();
+  e->pm->send(e->service_name, bytes.begin(), bytes.size());
+  if (e->write) {
+    e->to_write.push(words);
+  } else {
+    delete words;
+  }
+}
+
+// TODO: writing should be moved to loggerd
+void VideoEncoder::write_handler(VideoEncoder *e, const char *path) {
+  VideoWriter writer(path, e->filename, e->codec != cereal::EncodeIndex::Type::FULL_H_E_V_C, e->out_width, e->out_height, e->fps, e->codec);
+
+  bool first = true;
+  kj::Array<capnp::word>* out_buf;
+  while ((out_buf = e->to_write.pop())) {
+    capnp::FlatArrayMessageReader cmsg(*out_buf);
+    cereal::Event::Reader event = cmsg.getRoot<cereal::Event>();
+
+    auto edata = (e->type == DriverCam) ? event.getDriverEncodeData() :
+      ((e->type == WideRoadCam) ? event.getWideRoadEncodeData() :
+      (e->in_width == e->out_width ? event.getRoadEncodeData() : event.getQRoadEncodeData()));
+    auto idx = edata.getIdx();
+    auto flags = idx.getFlags();
+
+    if (first) {
+      assert(flags & V4L2_BUF_FLAG_KEYFRAME);
+      auto header = edata.getHeader();
+      writer.write((uint8_t *)header.begin(), header.size(), idx.getTimestampEof()/1000, true, false);
+      first = false;
+    }
+
+    // dangerous cast from const, but should be fine
+    auto data = edata.getData();
+    if (data.size() > 0) {
+      writer.write((uint8_t *)data.begin(), data.size(), idx.getTimestampEof()/1000, false, flags & V4L2_BUF_FLAG_KEYFRAME);
+    }
+
+    // free the data
+    delete out_buf;
+  }
+
+  // VideoWriter is freed on out of scope
+}

selfdrive/loggerd/encoder/encoder.h

@@ -0,0 +1,60 @@
+#pragma once
+
+#include <cassert>
+#include <cstdint>
+#include <thread>
+
+#include "cereal/messaging/messaging.h"
+#include "cereal/visionipc/visionipc.h"
+#include "selfdrive/common/queue.h"
+#include "selfdrive/loggerd/encoder/video_writer.h"
+#include "selfdrive/camerad/cameras/camera_common.h"
+
+#define V4L2_BUF_FLAG_KEYFRAME 8
+
+class VideoEncoder {
+public:
+  VideoEncoder(const char* filename, CameraType type, int in_width, int in_height, int fps,
+              int bitrate, cereal::EncodeIndex::Type codec, int out_width, int out_height, bool write)
+  : filename(filename), type(type), in_width(in_width), in_height(in_height), fps(fps),
+    bitrate(bitrate), codec(codec), out_width(out_width), out_height(out_height), write(write) { }
+  virtual ~VideoEncoder();
+  virtual int encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const uint8_t *v_ptr,
+                           int in_width, int in_height, VisionIpcBufExtra *extra) = 0;
+  virtual void encoder_open(const char* path) = 0;
+  virtual void encoder_close() = 0;
+
+  void publisher_init();
+  static void publisher_publish(VideoEncoder *e, int segment_num, uint32_t idx, VisionIpcBufExtra &extra, unsigned int flags, kj::ArrayPtr<capnp::byte> header, kj::ArrayPtr<capnp::byte> dat);
+
+  void writer_open(const char* path) {
+    if (this->write) write_handler_thread = std::thread(VideoEncoder::write_handler, this, path);
+  }
+
+  void writer_close() {
+    if (this->write) {
+      to_write.push(NULL);
+      write_handler_thread.join();
+    }
+    assert(to_write.empty());
+  }
+
+protected:
+  bool write;
+  const char* filename;
+  int in_width, in_height;
+  int out_width, out_height, fps;
+  int bitrate;
+  cereal::EncodeIndex::Type codec;
+  CameraType type;
+
+private:
+  // publishing
+  std::unique_ptr<PubMaster> pm;
+  const char *service_name;
+
+  // writing support
+  static void write_handler(VideoEncoder *e, const char *path);
+  std::thread write_handler_thread;
+  SafeQueue<kj::Array<capnp::word>* > to_write;
+};

selfdrive/loggerd/encoder/ffmpeg_encoder.cc

@@ -1,6 +1,6 @@
 #pragma clang diagnostic ignored "-Wdeprecated-declarations"
 
-#include "selfdrive/loggerd/raw_logger.h"
+#include "selfdrive/loggerd/encoder/ffmpeg_encoder.h"
 
 #include <fcntl.h>
 #include <unistd.h>
@@ -22,10 +22,9 @@ extern "C" {
 #include "selfdrive/common/swaglog.h"
 #include "selfdrive/common/util.h"
 
-RawLogger::RawLogger(const char* filename, CameraType type, int in_width, int in_height, int fps,
-                     int bitrate, bool h265, int out_width, int out_height, bool write)
-  : in_width_(in_width), in_height_(in_height), filename(filename), fps(fps) {
-  // TODO: respect write arg
+const int env_debug_encoder = (getenv("DEBUG_ENCODER") != NULL) ? atoi(getenv("DEBUG_ENCODER")) : 0;
+
+void FfmpegEncoder::encoder_init() {
   frame = av_frame_alloc();
   assert(frame);
   frame->format = AV_PIX_FMT_YUV420P;
@@ -38,30 +37,43 @@ RawLogger::RawLogger(const char* filename, CameraType type, int in_width, int in
   if (in_width != out_width || in_height != out_height) {
     downscale_buf.resize(out_width * out_height * 3 / 2);
   }
+
+  publisher_init();
 }
 
-RawLogger::~RawLogger() {
+FfmpegEncoder::~FfmpegEncoder() {
   encoder_close();
   av_frame_free(&frame);
 }
 
-void RawLogger::encoder_open(const char* path) {
-  writer = new VideoWriter(path, this->filename, true, frame->width, frame->height, this->fps, false, true);
-  // write the header
-  writer->write(NULL, 0, 0, true, false);
+void FfmpegEncoder::encoder_open(const char* path) {
+  AVCodec *codec = avcodec_find_encoder(AV_CODEC_ID_FFVHUFF);
+
+  this->codec_ctx = avcodec_alloc_context3(codec);
+  assert(this->codec_ctx);
+  this->codec_ctx->width = frame->width;
+  this->codec_ctx->height = frame->height;
+  this->codec_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
+  this->codec_ctx->time_base = (AVRational){ 1, fps };
+  int err = avcodec_open2(this->codec_ctx, codec, NULL);
+  assert(err >= 0);
+
+  writer_open(path);
   is_open = true;
+  segment_num++;
+  counter = 0;
 }
 
-void RawLogger::encoder_close() {
+void FfmpegEncoder::encoder_close() {
   if (!is_open) return;
-  delete writer;
+  writer_close();
   is_open = false;
 }
 
-int RawLogger::encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const uint8_t *v_ptr,
-                            int in_width, int in_height, VisionIpcBufExtra *extra) {
-  assert(in_width == this->in_width_);
-  assert(in_height == this->in_height_);
+int FfmpegEncoder::encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const uint8_t *v_ptr,
+                                int in_width_, int in_height_, VisionIpcBufExtra *extra) {
+  assert(in_width_ == this->in_width);
+  assert(in_height_ == this->in_height);
 
   if (downscale_buf.size() > 0) {
     uint8_t *out_y = downscale_buf.data();
@@ -88,7 +100,7 @@ int RawLogger::encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const ui
 
   int ret = counter;
 
-  int err = avcodec_send_frame(writer->codec_ctx, frame);
+  int err = avcodec_send_frame(this->codec_ctx, frame);
   if (err < 0) {
     LOGE("avcodec_send_frame error %d", err);
     ret = -1;
@@ -99,7 +111,7 @@ int RawLogger::encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const ui
   pkt.data = NULL;
   pkt.size = 0;
   while (ret >= 0) {
-    err = avcodec_receive_packet(writer->codec_ctx, &pkt);
+    err = avcodec_receive_packet(this->codec_ctx, &pkt);
     if (err == AVERROR_EOF) {
       break;
     } else if (err == AVERROR(EAGAIN)) {
@@ -112,7 +124,15 @@ int RawLogger::encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const ui
       break;
     }
 
-    writer->write(pkt.data, pkt.size, pkt.pts, false, pkt.flags & AV_PKT_FLAG_KEY);
+    if (env_debug_encoder) {
+      printf("%20s got %8d bytes flags %8x idx %4d id %8d\n", this->filename, pkt.size, pkt.flags, counter, extra->frame_id);
+    }
+
+    publisher_publish(this, segment_num, counter, *extra,
+      (pkt.flags & AV_PKT_FLAG_KEY) ? V4L2_BUF_FLAG_KEYFRAME : 0,
+      kj::arrayPtr<capnp::byte>(pkt.data, (size_t)0), // TODO: get the header
+      kj::arrayPtr<capnp::byte>(pkt.data, pkt.size));
+
     counter++;
   }
   av_packet_unref(&pkt);

selfdrive/loggerd/encoder/ffmpeg_encoder.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include <cstdio>
+#include <cstdlib>
+#include <string>
+#include <vector>
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libavutil/imgutils.h>
+}
+
+#include "selfdrive/loggerd/encoder/encoder.h"
+#include "selfdrive/loggerd/loggerd.h"
+#include "selfdrive/loggerd/encoder/video_writer.h"
+
+class FfmpegEncoder : public VideoEncoder {
+ public:
+  FfmpegEncoder(const char* filename, CameraType type, int in_width, int in_height, int fps,
+                int bitrate, cereal::EncodeIndex::Type codec, int out_width, int out_height, bool write) :
+                VideoEncoder(filename, type, in_width, in_height, fps, bitrate, cereal::EncodeIndex::Type::BIG_BOX_LOSSLESS, out_width, out_height, write) { encoder_init(); }
+  ~FfmpegEncoder();
+  void encoder_init();
+  int encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const uint8_t *v_ptr,
+                   int in_width_, int in_height_, VisionIpcBufExtra *extra);
+  void encoder_open(const char* path);
+  void encoder_close();
+
+private:
+  int segment_num = -1;
+  int counter = 0;
+  bool is_open = false;
+
+  AVCodecContext *codec_ctx;
+  AVFrame *frame = NULL;
+  std::vector<uint8_t> downscale_buf;
+};

selfdrive/loggerd/encoder/v4l_encoder.cc

@@ -2,7 +2,7 @@
 #include <sys/ioctl.h>
 #include <poll.h>
 
-#include "selfdrive/loggerd/v4l_encoder.h"
+#include "selfdrive/loggerd/encoder/v4l_encoder.h"
 #include "selfdrive/common/util.h"
 #include "selfdrive/common/timing.h"
 
@@ -67,42 +67,6 @@ static void request_buffers(int fd, v4l2_buf_type buf_type, unsigned int count)
   checked_ioctl(fd, VIDIOC_REQBUFS, &reqbuf);
 }
 
-// TODO: writing should be moved to loggerd
-void V4LEncoder::write_handler(V4LEncoder *e, const char *path) {
-  VideoWriter writer(path, e->filename, !e->h265, e->width, e->height, e->fps, e->h265, false);
-
-  bool first = true;
-  kj::Array<capnp::word>* out_buf;
-  while ((out_buf = e->to_write.pop())) {
-    capnp::FlatArrayMessageReader cmsg(*out_buf);
-    cereal::Event::Reader event = cmsg.getRoot<cereal::Event>();
-
-    auto edata = (e->type == DriverCam) ? event.getDriverEncodeData() :
-      ((e->type == WideRoadCam) ? event.getWideRoadEncodeData() :
-      (e->h265 ? event.getRoadEncodeData() : event.getQRoadEncodeData()));
-    auto idx = edata.getIdx();
-    auto flags = idx.getFlags();
-
-    if (first) {
-      assert(flags & V4L2_BUF_FLAG_KEYFRAME);
-      auto header = edata.getHeader();
-      writer.write((uint8_t *)header.begin(), header.size(), idx.getTimestampEof()/1000, true, false);
-      first = false;
-    }
-
-    // dangerous cast from const, but should be fine
-    auto data = edata.getData();
-    if (data.size() > 0) {
-      writer.write((uint8_t *)data.begin(), data.size(), idx.getTimestampEof()/1000, false, flags & V4L2_BUF_FLAG_KEYFRAME);
-    }
-
-    // free the data
-    delete out_buf;
-  }
-
-  // VideoWriter is freed on out of scope
-}
-
 void V4LEncoder::dequeue_handler(V4LEncoder *e) {
   std::string dequeue_thread_name = "dq-"+std::string(e->filename);
   util::set_thread_name(dequeue_thread_name.c_str());
@@ -138,7 +102,6 @@ void V4LEncoder::dequeue_handler(V4LEncoder *e) {
 
       // eof packet, we exit
       if (flags & V4L2_QCOM_BUF_FLAG_EOS) {
-        if (e->write) e->to_write.push(NULL);
         exit = true;
       } else if (flags & V4L2_QCOM_BUF_FLAG_CODECCONFIG) {
         // save header
@@ -146,37 +109,9 @@ void V4LEncoder::dequeue_handler(V4LEncoder *e) {
       } else {
         VisionIpcBufExtra extra = e->extras.pop();
         assert(extra.timestamp_eof/1000 == ts); // stay in sync
-
         frame_id = extra.frame_id;
         ++idx;
-
-        // broadcast packet
-        MessageBuilder msg;
-        auto event = msg.initEvent(true);
-        auto edat = (e->type == DriverCam) ? event.initDriverEncodeData() :
-          ((e->type == WideRoadCam) ? event.initWideRoadEncodeData() :
-          (e->h265 ? event.initRoadEncodeData() : event.initQRoadEncodeData()));
-        auto edata = edat.initIdx();
-        edata.setFrameId(extra.frame_id);
-        edata.setTimestampSof(extra.timestamp_sof);
-        edata.setTimestampEof(extra.timestamp_eof);
-        edata.setType(e->h265 ? cereal::EncodeIndex::Type::FULL_H_E_V_C : cereal::EncodeIndex::Type::QCAMERA_H264);
-        edata.setEncodeId(idx);
-        edata.setSegmentNum(e->segment_num);
-        edata.setSegmentId(idx);
-        edata.setFlags(flags);
-        edata.setLen(bytesused);
-        edat.setData(kj::arrayPtr<capnp::byte>(buf, bytesused));
-        if (flags & V4L2_BUF_FLAG_KEYFRAME) edat.setHeader(header);
-
-        auto words = new kj::Array<capnp::word>(capnp::messageToFlatArray(msg));
-        auto bytes = words->asBytes();
-        e->pm->send(e->service_name, bytes.begin(), bytes.size());
-        if (e->write) {
-          e->to_write.push(words);
-        } else {
-          delete words;
-        }
+        e->publisher_publish(e, e->segment_num, idx, extra, flags, header, kj::arrayPtr<capnp::byte>(buf, bytesused));
       }
 
       if (env_debug_encoder) {
@@ -196,12 +131,7 @@ void V4LEncoder::dequeue_handler(V4LEncoder *e) {
   }
 }
 
-V4LEncoder::V4LEncoder(
-  const char* filename, CameraType type, int in_width, int in_height,
-  int fps, int bitrate, bool h265, int out_width, int out_height, bool write)
-  : type(type), in_width_(in_width), in_height_(in_height),
-    filename(filename), h265(h265),
-    width(out_width), height(out_height), fps(fps), write(write) {
+void V4LEncoder::encoder_init() {
   fd = open("/dev/v4l/by-path/platform-aa00000.qcom_vidc-video-index1", O_RDWR|O_NONBLOCK);
   assert(fd >= 0);
 
@@ -218,7 +148,7 @@ V4LEncoder::V4LEncoder(
         // downscales are free with v4l
         .width = (unsigned int)out_width,
         .height = (unsigned int)out_height,
-        .pixelformat = h265 ? V4L2_PIX_FMT_HEVC : V4L2_PIX_FMT_H264,
+        .pixelformat = (codec == cereal::EncodeIndex::Type::FULL_H_E_V_C) ? V4L2_PIX_FMT_HEVC : V4L2_PIX_FMT_H264,
         .field = V4L2_FIELD_ANY,
         .colorspace = V4L2_COLORSPACE_DEFAULT,
       }
@@ -272,7 +202,7 @@ V4LEncoder::V4LEncoder(
     }
   }
 
-  if (h265) {
+  if (codec == cereal::EncodeIndex::Type::FULL_H_E_V_C) {
     struct v4l2_control ctrls[] = {
       { .id = V4L2_CID_MPEG_VIDC_VIDEO_HEVC_PROFILE, .value = V4L2_MPEG_VIDC_VIDEO_HEVC_PROFILE_MAIN},
       { .id = V4L2_CID_MPEG_VIDC_VIDEO_HEVC_TIER_LEVEL, .value = V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_5},
@@ -321,23 +251,18 @@ V4LEncoder::V4LEncoder(
     free_buf_in.push(i);
   }
 
-  // publish
-  service_name = this->type == DriverCam ? "driverEncodeData" :
-    (this->type == WideRoadCam ? "wideRoadEncodeData" :
-    (this->h265 ? "roadEncodeData" : "qRoadEncodeData"));
-  pm.reset(new PubMaster({service_name}));
+  publisher_init();
 }
 
-
 void V4LEncoder::encoder_open(const char* path) {
   dequeue_handler_thread = std::thread(V4LEncoder::dequeue_handler, this);
-  if (this->write) write_handler_thread = std::thread(V4LEncoder::write_handler, this, path);
+  writer_open(path);
   this->is_open = true;
   this->counter = 0;
 }
 
 int V4LEncoder::encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const uint8_t *v_ptr,
-                  int in_width, int in_height, VisionIpcBufExtra *extra) {
+                  int in_width_, int in_height_, VisionIpcBufExtra *extra) {
   assert(in_width == in_width_);
   assert(in_height == in_height_);
   assert(is_open);
@@ -383,8 +308,7 @@ void V4LEncoder::encoder_close() {
     // join waits for V4L2_QCOM_BUF_FLAG_EOS
     dequeue_handler_thread.join();
     assert(extras.empty());
-    if (this->write) write_handler_thread.join();
-    assert(to_write.empty());
+    writer_close();
   }
   this->is_open = false;
 }

selfdrive/loggerd/encoder/v4l_encoder.h

@@ -1,17 +1,19 @@
 #pragma once
 
 #include "selfdrive/common/queue.h"
-#include "selfdrive/loggerd/encoder.h"
-#include "selfdrive/loggerd/loggerd.h"
-#include "selfdrive/loggerd/video_writer.h"
+#include "selfdrive/loggerd/encoder/encoder.h"
+#include "selfdrive/loggerd/encoder/video_writer.h"
 
 #define BUF_IN_COUNT 7
 #define BUF_OUT_COUNT 6
 
 class V4LEncoder : public VideoEncoder {
 public:
-  V4LEncoder(const char* filename, CameraType type, int width, int height, int fps, int bitrate, bool h265, int out_width, int out_height, bool write = true);
+  V4LEncoder(const char* filename, CameraType type, int in_width, int in_height, int fps,
+             int bitrate, cereal::EncodeIndex::Type codec, int out_width, int out_height, bool write) :
+             VideoEncoder(filename, type, in_width, in_height, fps, bitrate, codec, out_width, out_height, write) { encoder_init(); }
   ~V4LEncoder();
+  void encoder_init();
   int encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const uint8_t *v_ptr,
                    int in_width, int in_height, VisionIpcBufExtra *extra);
   void encoder_open(const char* path);
@@ -19,16 +21,11 @@ public:
 private:
   int fd;
 
-  const char* filename;
-  CameraType type;
-  unsigned int in_width_, in_height_;
-  bool h265;
   bool is_open = false;
   int segment_num = -1;
   int counter = 0;
 
-  std::unique_ptr<PubMaster> pm;
-  const char *service_name;
+  SafeQueue<VisionIpcBufExtra> extras;
 
   static void dequeue_handler(V4LEncoder *e);
   std::thread dequeue_handler_thread;
@@ -36,13 +33,4 @@ private:
   VisionBuf buf_in[BUF_IN_COUNT];
   VisionBuf buf_out[BUF_OUT_COUNT];
   SafeQueue<unsigned int> free_buf_in;
-
-  SafeQueue<VisionIpcBufExtra> extras;
-
-  // writing support
-  int width, height, fps;
-  bool write;
-  static void write_handler(V4LEncoder *e, const char *path);
-  std::thread write_handler_thread;
-  SafeQueue<kj::Array<capnp::word>* > to_write;
 };

selfdrive/loggerd/encoder/video_writer.cc

@@ -3,12 +3,13 @@
 #include <cassert>
 #include <cstdlib>
 
-#include "selfdrive/loggerd/video_writer.h"
+#include "selfdrive/loggerd/encoder/video_writer.h"
 #include "selfdrive/common/swaglog.h"
 #include "selfdrive/common/util.h"
 
-VideoWriter::VideoWriter(const char *path, const char *filename, bool remuxing, int width, int height, int fps, bool h265, bool raw)
-  : remuxing(remuxing), raw(raw) {
+VideoWriter::VideoWriter(const char *path, const char *filename, bool remuxing, int width, int height, int fps, cereal::EncodeIndex::Type codec)
+  : remuxing(remuxing) {
+  raw = codec == cereal::EncodeIndex::Type::BIG_BOX_LOSSLESS;
   vid_path = util::string_format("%s/%s", path, filename);
   lock_path = util::string_format("%s/%s.lock", path, filename);
 
@@ -24,25 +25,25 @@ VideoWriter::VideoWriter(const char *path, const char *filename, bool remuxing,
     // set codec correctly. needed?
     av_register_all();
 
-    AVCodec *codec = NULL;
-    assert(!h265);
-    codec = avcodec_find_encoder(raw ? AV_CODEC_ID_FFVHUFF : AV_CODEC_ID_H264);
-    assert(codec);
+    AVCodec *avcodec = NULL;
+    assert(codec != cereal::EncodeIndex::Type::FULL_H_E_V_C);
+    avcodec = avcodec_find_encoder(raw ? AV_CODEC_ID_FFVHUFF : AV_CODEC_ID_H264);
+    assert(avcodec);
 
-    this->codec_ctx = avcodec_alloc_context3(codec);
+    this->codec_ctx = avcodec_alloc_context3(avcodec);
     assert(this->codec_ctx);
     this->codec_ctx->width = width;
     this->codec_ctx->height = height;
     this->codec_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
     this->codec_ctx->time_base = (AVRational){ 1, fps };
 
-    if (raw) {
-      // since the codec is actually used, we open it
-      int err = avcodec_open2(this->codec_ctx, codec, NULL);
+    if (codec == cereal::EncodeIndex::Type::BIG_BOX_LOSSLESS) {
+      // without this, there's just noise
+      int err = avcodec_open2(this->codec_ctx, avcodec, NULL);
       assert(err >= 0);
     }
 
-    this->out_stream = avformat_new_stream(this->ofmt_ctx, raw ? codec : NULL);
+    this->out_stream = avformat_new_stream(this->ofmt_ctx, raw ? avcodec : NULL);
     assert(this->out_stream);
 
     int err = avio_open(&this->ofmt_ctx->pb, this->vid_path.c_str(), AVIO_FLAG_WRITE);
@@ -64,7 +65,7 @@ void VideoWriter::write(uint8_t *data, int len, long long timestamp, bool codecc
 
   if (remuxing) {
     if (codecconfig) {
-      if (data) {
+      if (len > 0) {
         codec_ctx->extradata = (uint8_t*)av_mallocz(len + AV_INPUT_BUFFER_PADDING_SIZE);
         codec_ctx->extradata_size = len;
         memcpy(codec_ctx->extradata, data, len);

selfdrive/loggerd/encoder/video_writer.h

@@ -6,17 +6,19 @@ extern "C" {
 #include <libavformat/avformat.h>
 }
 
+#include "cereal/messaging/messaging.h"
+
 class VideoWriter {
 public:
-  VideoWriter(const char *path, const char *filename, bool remuxing, int width, int height, int fps, bool h265, bool raw);
+  VideoWriter(const char *path, const char *filename, bool remuxing, int width, int height, int fps, cereal::EncodeIndex::Type codec);
   void write(uint8_t *data, int len, long long timestamp, bool codecconfig, bool keyframe);
   ~VideoWriter();
-  AVCodecContext *codec_ctx;
 private:
   std::string vid_path, lock_path;
 
   FILE *of = nullptr;
 
+  AVCodecContext *codec_ctx;
   AVFormatContext *ofmt_ctx;
   AVStream *out_stream;
   bool remuxing, raw;

selfdrive/loggerd/encoderd.cc

@@ -54,12 +54,14 @@ void encoder_thread(EncoderdState *s, const LogCameraInfo &cam_info) {
 
       // main encoder
       encoders.push_back(new Encoder(cam_info.filename, cam_info.type, buf_info.width, buf_info.height,
-                                     cam_info.fps, cam_info.bitrate, cam_info.is_h265,
+                                     cam_info.fps, cam_info.bitrate,
+                                     cam_info.is_h265 ? cereal::EncodeIndex::Type::FULL_H_E_V_C : cereal::EncodeIndex::Type::QCAMERA_H264,
                                      buf_info.width, buf_info.height, false));
       // qcamera encoder
       if (cam_info.has_qcamera) {
         encoders.push_back(new Encoder(qcam_info.filename, cam_info.type, buf_info.width, buf_info.height,
-                                       qcam_info.fps, qcam_info.bitrate, qcam_info.is_h265,
+                                       qcam_info.fps, qcam_info.bitrate,
+                                       qcam_info.is_h265 ? cereal::EncodeIndex::Type::FULL_H_E_V_C : cereal::EncodeIndex::Type::QCAMERA_H264,
                                        qcam_info.frame_width, qcam_info.frame_height, false));
       }
     }

selfdrive/loggerd/loggerd.cc

@@ -65,13 +65,15 @@ void encoder_thread(LoggerdState *s, const LogCameraInfo &cam_info) {
 
       // main encoder
       encoders.push_back(new Encoder(cam_info.filename, cam_info.type, buf_info.width, buf_info.height,
-                                     cam_info.fps, cam_info.bitrate, cam_info.is_h265,
+                                     cam_info.fps, cam_info.bitrate,
+                                     cam_info.is_h265 ? cereal::EncodeIndex::Type::FULL_H_E_V_C : cereal::EncodeIndex::Type::QCAMERA_H264,
                                      buf_info.width, buf_info.height, cam_info.record));
       // qcamera encoder
       if (cam_info.has_qcamera) {
         encoders.push_back(new Encoder(qcam_info.filename, cam_info.type, buf_info.width, buf_info.height,
-                                       qcam_info.fps, qcam_info.bitrate, qcam_info.is_h265,
-                                       qcam_info.frame_width, qcam_info.frame_height));
+                                       qcam_info.fps, qcam_info.bitrate,
+                                       qcam_info.is_h265 ? cereal::EncodeIndex::Type::FULL_H_E_V_C : cereal::EncodeIndex::Type::QCAMERA_H264,
+                                       qcam_info.frame_width, qcam_info.frame_height, true));
       }
     }
 

selfdrive/loggerd/loggerd.h

@@ -22,14 +22,14 @@
 #include "selfdrive/common/util.h"
 #include "selfdrive/hardware/hw.h"
 
-#include "selfdrive/loggerd/encoder.h"
+#include "selfdrive/loggerd/encoder/encoder.h"
 #include "selfdrive/loggerd/logger.h"
 #ifdef QCOM2
-#include "selfdrive/loggerd/v4l_encoder.h"
+#include "selfdrive/loggerd/encoder/v4l_encoder.h"
 #define Encoder V4LEncoder
 #else
-#include "selfdrive/loggerd/raw_logger.h"
-#define Encoder RawLogger
+#include "selfdrive/loggerd/encoder/ffmpeg_encoder.h"
+#define Encoder FfmpegEncoder
 #endif
 
 constexpr int MAIN_FPS = 20;

selfdrive/loggerd/raw_logger.h

@@ -1,41 +0,0 @@
-#pragma once
-
-#include <cstdio>
-#include <cstdlib>
-#include <string>
-#include <vector>
-
-extern "C" {
-#include <libavcodec/avcodec.h>
-#include <libavformat/avformat.h>
-#include <libavutil/imgutils.h>
-}
-
-#include "selfdrive/loggerd/encoder.h"
-#include "selfdrive/loggerd/loggerd.h"
-#include "selfdrive/loggerd/video_writer.h"
-
-class RawLogger : public VideoEncoder {
- public:
-  RawLogger(const char* filename, CameraType type, int in_width, int in_height, int fps,
-            int bitrate, bool h265, int out_width, int out_height, bool write = true);
-  ~RawLogger();
-  int encode_frame(const uint8_t *y_ptr, const uint8_t *u_ptr, const uint8_t *v_ptr,
-                   int in_width, int in_height, VisionIpcBufExtra *extra);
-  void encoder_open(const char* path);
-  void encoder_close();
-
-private:
-  const char* filename;
-  //bool write;
-  int fps;
-  int counter = 0;
-  bool is_open = false;
-
-  int in_width_, in_height_;
-
-  AVFrame *frame = NULL;
-  std::vector<uint8_t> downscale_buf;
-
-  VideoWriter *writer = NULL;
-};

selfdrive/loggerd/remote_encoder.cc

@@ -40,7 +40,7 @@ int handle_encoder_msg(LoggerdState *s, Message *msg, std::string &name, struct
       re.writer.reset(new VideoWriter(s->segment_path,
         cam_info.filename, !cam_info.is_h265,
         cam_info.frame_width, cam_info.frame_height,
-        cam_info.fps, cam_info.is_h265, false));
+        cam_info.fps, cam_info.is_h265 ? cereal::EncodeIndex::Type::FULL_H_E_V_C : cereal::EncodeIndex::Type::QCAMERA_H264));
       // write the header
       auto header = edata.getHeader();
       re.writer->write((uint8_t *)header.begin(), header.size(), idx.getTimestampEof()/1000, true, false);

selfdrive/loggerd/remote_encoder.h

@@ -1,5 +1,4 @@
-#include "selfdrive/loggerd/video_writer.h"
-#define V4L2_BUF_FLAG_KEYFRAME 0x00000008
+#include "selfdrive/loggerd/encoder/video_writer.h"
 
 struct RemoteEncoder {
   std::unique_ptr<VideoWriter> writer;

tools/camerastream/compressed_vipc.py

@@ -71,7 +71,7 @@ def decoder(addr, sock_name, vipc_server, vst, nvidia):
         assert len(frames) == 1
         img_yuv = frames[0].to_ndarray(format=av.video.format.VideoFormat('yuv420p'))
 
-      vipc_server.send(vst, img_yuv.flatten().data, cnt, 0, 0)
+      vipc_server.send(vst, img_yuv.flatten().data, cnt, cnt*1e9/20, cnt*1e9/20)
       cnt += 1
 
       pc_latency = (time.monotonic()-time_q[0])*1000