replay: cleanup unused RGB buffers (#24537)

* replay: cleanup unused RGB buffers

* update replay cam

* little more
old-commit-hash: ab75c53792

selfdrive/camerad/cameras/camera_replay.cc

@@ -47,17 +47,16 @@ void camera_close(CameraState *s) {
 void run_camera(CameraState *s) {
   uint32_t stream_frame_id = 0, frame_id = 0;
   size_t buf_idx = 0;
-  std::unique_ptr<uint8_t[]> rgb_buf = std::make_unique<uint8_t[]>(s->frame->getRGBSize());
   std::unique_ptr<uint8_t[]> yuv_buf = std::make_unique<uint8_t[]>(s->frame->getYUVSize());
   while (!do_exit) {
     if (stream_frame_id == s->frame->getFrameCount()) {
       // loop stream
       stream_frame_id = 0;
     }
-    if (s->frame->get(stream_frame_id++, rgb_buf.get(), yuv_buf.get())) {
+    if (s->frame->get(stream_frame_id++, yuv_buf.get())) {
       s->buf.camera_bufs_metadata[buf_idx] = {.frame_id = frame_id};
       auto &buf = s->buf.camera_bufs[buf_idx];
-      CL_CHECK(clEnqueueWriteBuffer(buf.copy_q, buf.buf_cl, CL_TRUE, 0, s->frame->getRGBSize(), rgb_buf.get(), 0, NULL, NULL));
+      CL_CHECK(clEnqueueWriteBuffer(buf.copy_q, buf.buf_cl, CL_TRUE, 0, s->frame->getYUVSize(), yuv_buf.get(), 0, NULL, NULL));
       s->buf.queue(buf_idx);
       ++frame_id;
       buf_idx = (buf_idx + 1) % FRAME_BUF_COUNT;

selfdrive/test/process_replay/regen.py

@@ -165,8 +165,6 @@ def replay_cameras(lr, frs):
     p.append(multiprocessing.Process(target=replay_camera,
                                      args=(s, stream, dt, vs, frames, size, use_extra_client)))
 
-  # hack to make UI work
-  vs.create_buffers(VisionStreamType.VISION_STREAM_RGB_ROAD, 4, True, eon_f_frame_size[0], eon_f_frame_size[1])
   vs.start_listener()
   return vs, p
 

selfdrive/ui/replay/camera.cc

@@ -3,7 +3,7 @@
 
 #include <cassert>
 
-CameraServer::CameraServer(std::pair<int, int> camera_size[MAX_CAMERAS], bool send_yuv) : send_yuv(send_yuv) {
+CameraServer::CameraServer(std::pair<int, int> camera_size[MAX_CAMERAS]) {
   for (int i = 0; i < MAX_CAMERAS; ++i) {
     std::tie(cameras_[i].width, cameras_[i].height) = camera_size[i];
   }
@@ -25,10 +25,7 @@ void CameraServer::startVipcServer() {
   for (auto &cam : cameras_) {
     if (cam.width > 0 && cam.height > 0) {
       rInfo("camera[%d] frame size %dx%d", cam.type, cam.width, cam.height);
-      vipc_server_->create_buffers(cam.rgb_type, UI_BUF_COUNT, true, cam.width, cam.height);
-      if (send_yuv) {
-        vipc_server_->create_buffers(cam.yuv_type, YUV_BUFFER_COUNT, false, cam.width, cam.height);
-      }
+      vipc_server_->create_buffers(cam.stream_type, YUV_BUFFER_COUNT, false, cam.width, cam.height);
       if (!cam.thread.joinable()) {
         cam.thread = std::thread(&CameraServer::cameraThread, this, std::ref(cam));
       }
@@ -39,10 +36,9 @@ void CameraServer::startVipcServer() {
 
 void CameraServer::cameraThread(Camera &cam) {
   auto read_frame = [&](FrameReader *fr, int frame_id) {
-    VisionBuf *rgb_buf = vipc_server_->get_buffer(cam.rgb_type);
-    VisionBuf *yuv_buf = send_yuv ? vipc_server_->get_buffer(cam.yuv_type) : nullptr;
-    bool ret = fr->get(frame_id, (uint8_t *)rgb_buf->addr, yuv_buf ? (uint8_t *)yuv_buf->addr : nullptr);
-    return ret ? std::pair{rgb_buf, yuv_buf} : std::pair{nullptr, nullptr};
+    VisionBuf *yuv_buf = vipc_server_->get_buffer(cam.stream_type);
+    bool ret = fr->get(frame_id, yuv_buf ? (uint8_t *)yuv_buf->addr : nullptr);
+    return ret ? yuv_buf : nullptr;
   };
 
   while (true) {
@@ -51,15 +47,14 @@ void CameraServer::cameraThread(Camera &cam) {
 
     const int id = eidx.getSegmentId();
     bool prefetched = (id == cam.cached_id && eidx.getSegmentNum() == cam.cached_seg);
-    auto [rgb, yuv] = prefetched ? cam.cached_buf : read_frame(fr, id);
-    if (rgb || yuv) {
+    auto yuv = prefetched ? cam.cached_buf : read_frame(fr, id);
+    if (yuv) {
       VisionIpcBufExtra extra = {
           .frame_id = eidx.getFrameId(),
           .timestamp_sof = eidx.getTimestampSof(),
           .timestamp_eof = eidx.getTimestampEof(),
       };
-      if (rgb) vipc_server_->send(rgb, &extra, false);
-      if (yuv) vipc_server_->send(yuv, &extra, false);
+      vipc_server_->send(yuv, &extra, false);
     } else {
       rError("camera[%d] failed to get frame:", cam.type, eidx.getSegmentId());
     }

selfdrive/ui/replay/camera.h

@@ -8,7 +8,7 @@
 
 class CameraServer {
 public:
-  CameraServer(std::pair<int, int> camera_size[MAX_CAMERAS] = nullptr, bool send_yuv = false);
+  CameraServer(std::pair<int, int> camera_size[MAX_CAMERAS] = nullptr);
   ~CameraServer();
   void pushFrame(CameraType type, FrameReader* fr, const cereal::EncodeIndex::Reader& eidx);
   void waitForSent();
@@ -16,25 +16,23 @@ public:
 protected:
   struct Camera {
     CameraType type;
-    VisionStreamType rgb_type;
-    VisionStreamType yuv_type;
+    VisionStreamType stream_type;
     int width;
     int height;
     std::thread thread;
     SafeQueue<std::pair<FrameReader*, const cereal::EncodeIndex::Reader>> queue;
     int cached_id = -1;
     int cached_seg = -1;
-    std::pair<VisionBuf *, VisionBuf*> cached_buf;
+    VisionBuf * cached_buf;
   };
   void startVipcServer();
   void cameraThread(Camera &cam);
 
   Camera cameras_[MAX_CAMERAS] = {
-      {.type = RoadCam, .rgb_type = VISION_STREAM_RGB_ROAD, .yuv_type = VISION_STREAM_ROAD},
-      {.type = DriverCam, .rgb_type = VISION_STREAM_RGB_DRIVER, .yuv_type = VISION_STREAM_DRIVER},
-      {.type = WideRoadCam, .rgb_type = VISION_STREAM_RGB_WIDE_ROAD, .yuv_type = VISION_STREAM_WIDE_ROAD},
+      {.type = RoadCam, .stream_type = VISION_STREAM_ROAD},
+      {.type = DriverCam, .stream_type = VISION_STREAM_DRIVER},
+      {.type = WideRoadCam, .stream_type = VISION_STREAM_WIDE_ROAD},
   };
   std::atomic<int> publishing_ = 0;
   std::unique_ptr<VisionIpcServer> vipc_server_;
-  bool send_yuv;
 };

selfdrive/ui/replay/framereader.cc

@@ -170,15 +170,15 @@ bool FrameReader::initHardwareDecoder(AVHWDeviceType hw_device_type) {
   return true;
 }
 
-bool FrameReader::get(int idx, uint8_t *rgb, uint8_t *yuv) {
-  assert(rgb || yuv);
+bool FrameReader::get(int idx, uint8_t *yuv) {
+  assert(yuv != nullptr);
   if (!valid_ || idx < 0 || idx >= packets.size()) {
     return false;
   }
-  return decode(idx, rgb, yuv);
+  return decode(idx, yuv);
 }
 
-bool FrameReader::decode(int idx, uint8_t *rgb, uint8_t *yuv) {
+bool FrameReader::decode(int idx, uint8_t *yuv) {
   int from_idx = idx;
   if (idx != prev_idx + 1 && key_frames_count_ > 1) {
     // seeking to the nearest key frame
@@ -194,7 +194,7 @@ bool FrameReader::decode(int idx, uint8_t *rgb, uint8_t *yuv) {
   for (int i = from_idx; i <= idx; ++i) {
     AVFrame *f = decodeFrame(packets[i]);
     if (f && i == idx) {
-      return copyBuffers(f, rgb, yuv);
+      return copyBuffers(f, yuv);
     }
   }
   return false;
@@ -226,29 +226,21 @@ AVFrame *FrameReader::decodeFrame(AVPacket *pkt) {
   }
 }
 
-bool FrameReader::copyBuffers(AVFrame *f, uint8_t *rgb, uint8_t *yuv) {
+bool FrameReader::copyBuffers(AVFrame *f, uint8_t *yuv) {
   if (hw_pix_fmt == HW_PIX_FMT) {
     uint8_t *y = yuv ? yuv : nv12toyuv_buffer.data();
     uint8_t *u = y + width * height;
     uint8_t *v = u + (width / 2) * (height / 2);
     libyuv::NV12ToI420(f->data[0], f->linesize[0], f->data[1], f->linesize[1],
                        y, width, u, width / 2, v, width / 2, width, height);
-    libyuv::I420ToRGB24(y, width, u, width / 2, v, width / 2,
-                        rgb, aligned_width * 3, width, height);
   } else {
-    if (yuv) {
-      uint8_t *u = yuv + width * height;
-      uint8_t *v = u + (width / 2) * (height / 2);
-      libyuv::I420Copy(f->data[0], f->linesize[0],
-                       f->data[1], f->linesize[1],
-                       f->data[2], f->linesize[2],
-                       yuv, width, u, width / 2, v, width / 2,
-                       width, height);
-    }
-    libyuv::I420ToRGB24(f->data[0], f->linesize[0],
-                        f->data[1], f->linesize[1],
-                        f->data[2], f->linesize[2],
-                        rgb, aligned_width * 3, width, height);
+    uint8_t *u = yuv + width * height;
+    uint8_t *v = u + (width / 2) * (height / 2);
+    libyuv::I420Copy(f->data[0], f->linesize[0],
+                     f->data[1], f->linesize[1],
+                     f->data[2], f->linesize[2],
+                     yuv, width, u, width / 2, v, width / 2,
+                     width, height);
   }
   return true;
 }

selfdrive/ui/replay/framereader.h

@@ -19,10 +19,10 @@ class FrameReader {
 public:
   FrameReader();
   ~FrameReader();
-  bool load(const std::string &url, bool no_hw_decoder = false, std::atomic<bool> *abort = nullptr, bool local_cache = false, int chunk_size = -1, int retries = 0);
+  bool load(const std::string &url, bool no_hw_decoder = false, std::atomic<bool> *abort = nullptr, bool local_cache = false,
+            int chunk_size = -1, int retries = 0);
   bool load(const std::byte *data, size_t size, bool no_hw_decoder = false, std::atomic<bool> *abort = nullptr);
-  bool get(int idx, uint8_t *rgb, uint8_t *yuv);
-  int getRGBSize() const { return aligned_width * aligned_height * 3; }
+  bool get(int idx, uint8_t *yuv);
   int getYUVSize() const { return width * height * 3 / 2; }
   size_t getFrameCount() const { return packets.size(); }
   bool valid() const { return valid_; }
@@ -32,9 +32,9 @@ public:
 
 private:
   bool initHardwareDecoder(AVHWDeviceType hw_device_type);
-  bool decode(int idx, uint8_t *rgb, uint8_t *yuv);
+  bool decode(int idx, uint8_t *yuv);
   AVFrame * decodeFrame(AVPacket *pkt);
-  bool copyBuffers(AVFrame *f, uint8_t *rgb, uint8_t *yuv);
+  bool copyBuffers(AVFrame *f, uint8_t *yuv);
 
   std::vector<AVPacket*> packets;
   std::unique_ptr<AVFrame, AVFrameDeleter>av_frame_, hw_frame;

selfdrive/ui/replay/replay.cc

@@ -289,7 +289,7 @@ void Replay::startStream(const Segment *cur_segment) {
         camera_size[type] = {fr->width, fr->height};
       }
     }
-    camera_server_ = std::make_unique<CameraServer>(camera_size, true);
+    camera_server_ = std::make_unique<CameraServer>(camera_size);
   }
 
   // start stream thread

selfdrive/ui/replay/tests/test_replay.cc

@@ -102,11 +102,10 @@ void read_segment(int n, const SegmentFile &segment_file, uint32_t flags) {
       if (cam == RoadCam || cam == WideRoadCam) {
         REQUIRE(fr->getFrameCount() == 1200);
       }
-      std::unique_ptr<uint8_t[]> rgb_buf = std::make_unique<uint8_t[]>(fr->getRGBSize());
       std::unique_ptr<uint8_t[]> yuv_buf = std::make_unique<uint8_t[]>(fr->getYUVSize());
       // sequence get 100 frames
       for (int i = 0; i < 100; ++i) {
-        REQUIRE(fr->get(i, rgb_buf.get(), yuv_buf.get()));
+        REQUIRE(fr->get(i, yuv_buf.get()));
       }
     }
 

tools/sim/bridge.py

@@ -70,11 +70,7 @@ class Camerad:
     self.frame_wide_id = 0
     self.vipc_server = VisionIpcServer("camerad")
 
-    # TODO: remove RGB buffers once the last RGB vipc subscriber is removed
-    self.vipc_server.create_buffers(VisionStreamType.VISION_STREAM_RGB_ROAD, 4, True, W, H)
     self.vipc_server.create_buffers(VisionStreamType.VISION_STREAM_ROAD, 5, False, W, H)
-
-    self.vipc_server.create_buffers(VisionStreamType.VISION_STREAM_RGB_WIDE_ROAD, 4, True, W, H)
     self.vipc_server.create_buffers(VisionStreamType.VISION_STREAM_WIDE_ROAD, 5, False, W, H)
     self.vipc_server.start_listener()
 
@@ -92,16 +88,14 @@ class Camerad:
     self.Hdiv4 = H // 4 if (H % 4 == 0) else (H + (4 - H % 4)) // 4
 
   def cam_callback_road(self, image):
-    self._cam_callback(image, self.frame_road_id, 'roadCameraState',
-                       VisionStreamType.VISION_STREAM_RGB_ROAD, VisionStreamType.VISION_STREAM_ROAD)
+    self._cam_callback(image, self.frame_road_id, 'roadCameraState', VisionStreamType.VISION_STREAM_ROAD)
     self.frame_road_id += 1
 
   def cam_callback_wide_road(self, image):
-    self._cam_callback(image, self.frame_wide_id, 'wideRoadCameraState',
-                       VisionStreamType.VISION_STREAM_RGB_WIDE_ROAD, VisionStreamType.VISION_STREAM_WIDE_ROAD)
+    self._cam_callback(image, self.frame_wide_id, 'wideRoadCameraState', VisionStreamType.VISION_STREAM_WIDE_ROAD)
     self.frame_wide_id += 1
 
-  def _cam_callback(self, image, frame_id, pub_type, rgb_type, yuv_type):
+  def _cam_callback(self, image, frame_id, pub_type, yuv_type):
     img = np.frombuffer(image.raw_data, dtype=np.dtype("uint8"))
     img = np.reshape(img, (H, W, 4))
     img = img[:, :, [0, 1, 2]].copy()
@@ -114,8 +108,6 @@ class Camerad:
     yuv = np.resize(yuv_cl.get(), rgb.size // 2)
     eof = int(frame_id * 0.05 * 1e9)
 
-    # TODO: remove RGB send once the last RGB vipc subscriber is removed
-    self.vipc_server.send(rgb_type, img.tobytes(), frame_id, eof, eof)
     self.vipc_server.send(yuv_type, yuv.data.tobytes(), frame_id, eof, eof)
 
     dat = messaging.new_message(pub_type)