diff --git a/selfdrive/camerad/cameras/camera_replay.cc b/selfdrive/camerad/cameras/camera_replay.cc index 05bdfb923..56bda697c 100644 --- a/selfdrive/camerad/cameras/camera_replay.cc +++ b/selfdrive/camerad/cameras/camera_replay.cc @@ -47,17 +47,16 @@ void camera_close(CameraState *s) { void run_camera(CameraState *s) { uint32_t stream_frame_id = 0, frame_id = 0; size_t buf_idx = 0; - std::unique_ptr rgb_buf = std::make_unique(s->frame->getRGBSize()); std::unique_ptr yuv_buf = std::make_unique(s->frame->getYUVSize()); while (!do_exit) { if (stream_frame_id == s->frame->getFrameCount()) { // loop stream stream_frame_id = 0; } - if (s->frame->get(stream_frame_id++, rgb_buf.get(), yuv_buf.get())) { + if (s->frame->get(stream_frame_id++, yuv_buf.get())) { s->buf.camera_bufs_metadata[buf_idx] = {.frame_id = frame_id}; auto &buf = s->buf.camera_bufs[buf_idx]; - CL_CHECK(clEnqueueWriteBuffer(buf.copy_q, buf.buf_cl, CL_TRUE, 0, s->frame->getRGBSize(), rgb_buf.get(), 0, NULL, NULL)); + CL_CHECK(clEnqueueWriteBuffer(buf.copy_q, buf.buf_cl, CL_TRUE, 0, s->frame->getYUVSize(), yuv_buf.get(), 0, NULL, NULL)); s->buf.queue(buf_idx); ++frame_id; buf_idx = (buf_idx + 1) % FRAME_BUF_COUNT; diff --git a/selfdrive/test/process_replay/regen.py b/selfdrive/test/process_replay/regen.py index 9ec03f46b..3ceb98922 100755 --- a/selfdrive/test/process_replay/regen.py +++ b/selfdrive/test/process_replay/regen.py @@ -165,8 +165,6 @@ def replay_cameras(lr, frs): p.append(multiprocessing.Process(target=replay_camera, args=(s, stream, dt, vs, frames, size, use_extra_client))) - # hack to make UI work - vs.create_buffers(VisionStreamType.VISION_STREAM_RGB_ROAD, 4, True, eon_f_frame_size[0], eon_f_frame_size[1]) vs.start_listener() return vs, p diff --git a/selfdrive/ui/replay/camera.cc b/selfdrive/ui/replay/camera.cc index ab4f02e89..2e8b68a41 100644 --- a/selfdrive/ui/replay/camera.cc +++ b/selfdrive/ui/replay/camera.cc @@ -3,7 +3,7 @@ #include -CameraServer::CameraServer(std::pair camera_size[MAX_CAMERAS], bool send_yuv) : send_yuv(send_yuv) { +CameraServer::CameraServer(std::pair camera_size[MAX_CAMERAS]) { for (int i = 0; i < MAX_CAMERAS; ++i) { std::tie(cameras_[i].width, cameras_[i].height) = camera_size[i]; } @@ -25,10 +25,7 @@ void CameraServer::startVipcServer() { for (auto &cam : cameras_) { if (cam.width > 0 && cam.height > 0) { rInfo("camera[%d] frame size %dx%d", cam.type, cam.width, cam.height); - vipc_server_->create_buffers(cam.rgb_type, UI_BUF_COUNT, true, cam.width, cam.height); - if (send_yuv) { - vipc_server_->create_buffers(cam.yuv_type, YUV_BUFFER_COUNT, false, cam.width, cam.height); - } + vipc_server_->create_buffers(cam.stream_type, YUV_BUFFER_COUNT, false, cam.width, cam.height); if (!cam.thread.joinable()) { cam.thread = std::thread(&CameraServer::cameraThread, this, std::ref(cam)); } @@ -39,10 +36,9 @@ void CameraServer::startVipcServer() { void CameraServer::cameraThread(Camera &cam) { auto read_frame = [&](FrameReader *fr, int frame_id) { - VisionBuf *rgb_buf = vipc_server_->get_buffer(cam.rgb_type); - VisionBuf *yuv_buf = send_yuv ? vipc_server_->get_buffer(cam.yuv_type) : nullptr; - bool ret = fr->get(frame_id, (uint8_t *)rgb_buf->addr, yuv_buf ? (uint8_t *)yuv_buf->addr : nullptr); - return ret ? std::pair{rgb_buf, yuv_buf} : std::pair{nullptr, nullptr}; + VisionBuf *yuv_buf = vipc_server_->get_buffer(cam.stream_type); + bool ret = fr->get(frame_id, yuv_buf ? (uint8_t *)yuv_buf->addr : nullptr); + return ret ? yuv_buf : nullptr; }; while (true) { @@ -51,15 +47,14 @@ void CameraServer::cameraThread(Camera &cam) { const int id = eidx.getSegmentId(); bool prefetched = (id == cam.cached_id && eidx.getSegmentNum() == cam.cached_seg); - auto [rgb, yuv] = prefetched ? cam.cached_buf : read_frame(fr, id); - if (rgb || yuv) { + auto yuv = prefetched ? cam.cached_buf : read_frame(fr, id); + if (yuv) { VisionIpcBufExtra extra = { .frame_id = eidx.getFrameId(), .timestamp_sof = eidx.getTimestampSof(), .timestamp_eof = eidx.getTimestampEof(), }; - if (rgb) vipc_server_->send(rgb, &extra, false); - if (yuv) vipc_server_->send(yuv, &extra, false); + vipc_server_->send(yuv, &extra, false); } else { rError("camera[%d] failed to get frame:", cam.type, eidx.getSegmentId()); } diff --git a/selfdrive/ui/replay/camera.h b/selfdrive/ui/replay/camera.h index 753a059ce..84f0172d5 100644 --- a/selfdrive/ui/replay/camera.h +++ b/selfdrive/ui/replay/camera.h @@ -8,7 +8,7 @@ class CameraServer { public: - CameraServer(std::pair camera_size[MAX_CAMERAS] = nullptr, bool send_yuv = false); + CameraServer(std::pair camera_size[MAX_CAMERAS] = nullptr); ~CameraServer(); void pushFrame(CameraType type, FrameReader* fr, const cereal::EncodeIndex::Reader& eidx); void waitForSent(); @@ -16,25 +16,23 @@ public: protected: struct Camera { CameraType type; - VisionStreamType rgb_type; - VisionStreamType yuv_type; + VisionStreamType stream_type; int width; int height; std::thread thread; SafeQueue> queue; int cached_id = -1; int cached_seg = -1; - std::pair cached_buf; + VisionBuf * cached_buf; }; void startVipcServer(); void cameraThread(Camera &cam); Camera cameras_[MAX_CAMERAS] = { - {.type = RoadCam, .rgb_type = VISION_STREAM_RGB_ROAD, .yuv_type = VISION_STREAM_ROAD}, - {.type = DriverCam, .rgb_type = VISION_STREAM_RGB_DRIVER, .yuv_type = VISION_STREAM_DRIVER}, - {.type = WideRoadCam, .rgb_type = VISION_STREAM_RGB_WIDE_ROAD, .yuv_type = VISION_STREAM_WIDE_ROAD}, + {.type = RoadCam, .stream_type = VISION_STREAM_ROAD}, + {.type = DriverCam, .stream_type = VISION_STREAM_DRIVER}, + {.type = WideRoadCam, .stream_type = VISION_STREAM_WIDE_ROAD}, }; std::atomic publishing_ = 0; std::unique_ptr vipc_server_; - bool send_yuv; }; diff --git a/selfdrive/ui/replay/framereader.cc b/selfdrive/ui/replay/framereader.cc index 3186b4ff3..ca839e681 100644 --- a/selfdrive/ui/replay/framereader.cc +++ b/selfdrive/ui/replay/framereader.cc @@ -170,15 +170,15 @@ bool FrameReader::initHardwareDecoder(AVHWDeviceType hw_device_type) { return true; } -bool FrameReader::get(int idx, uint8_t *rgb, uint8_t *yuv) { - assert(rgb || yuv); +bool FrameReader::get(int idx, uint8_t *yuv) { + assert(yuv != nullptr); if (!valid_ || idx < 0 || idx >= packets.size()) { return false; } - return decode(idx, rgb, yuv); + return decode(idx, yuv); } -bool FrameReader::decode(int idx, uint8_t *rgb, uint8_t *yuv) { +bool FrameReader::decode(int idx, uint8_t *yuv) { int from_idx = idx; if (idx != prev_idx + 1 && key_frames_count_ > 1) { // seeking to the nearest key frame @@ -194,7 +194,7 @@ bool FrameReader::decode(int idx, uint8_t *rgb, uint8_t *yuv) { for (int i = from_idx; i <= idx; ++i) { AVFrame *f = decodeFrame(packets[i]); if (f && i == idx) { - return copyBuffers(f, rgb, yuv); + return copyBuffers(f, yuv); } } return false; @@ -226,29 +226,21 @@ AVFrame *FrameReader::decodeFrame(AVPacket *pkt) { } } -bool FrameReader::copyBuffers(AVFrame *f, uint8_t *rgb, uint8_t *yuv) { +bool FrameReader::copyBuffers(AVFrame *f, uint8_t *yuv) { if (hw_pix_fmt == HW_PIX_FMT) { uint8_t *y = yuv ? yuv : nv12toyuv_buffer.data(); uint8_t *u = y + width * height; uint8_t *v = u + (width / 2) * (height / 2); libyuv::NV12ToI420(f->data[0], f->linesize[0], f->data[1], f->linesize[1], y, width, u, width / 2, v, width / 2, width, height); - libyuv::I420ToRGB24(y, width, u, width / 2, v, width / 2, - rgb, aligned_width * 3, width, height); } else { - if (yuv) { - uint8_t *u = yuv + width * height; - uint8_t *v = u + (width / 2) * (height / 2); - libyuv::I420Copy(f->data[0], f->linesize[0], - f->data[1], f->linesize[1], - f->data[2], f->linesize[2], - yuv, width, u, width / 2, v, width / 2, - width, height); - } - libyuv::I420ToRGB24(f->data[0], f->linesize[0], - f->data[1], f->linesize[1], - f->data[2], f->linesize[2], - rgb, aligned_width * 3, width, height); + uint8_t *u = yuv + width * height; + uint8_t *v = u + (width / 2) * (height / 2); + libyuv::I420Copy(f->data[0], f->linesize[0], + f->data[1], f->linesize[1], + f->data[2], f->linesize[2], + yuv, width, u, width / 2, v, width / 2, + width, height); } return true; } diff --git a/selfdrive/ui/replay/framereader.h b/selfdrive/ui/replay/framereader.h index b58aca5f7..443636e27 100644 --- a/selfdrive/ui/replay/framereader.h +++ b/selfdrive/ui/replay/framereader.h @@ -19,10 +19,10 @@ class FrameReader { public: FrameReader(); ~FrameReader(); - bool load(const std::string &url, bool no_hw_decoder = false, std::atomic *abort = nullptr, bool local_cache = false, int chunk_size = -1, int retries = 0); + bool load(const std::string &url, bool no_hw_decoder = false, std::atomic *abort = nullptr, bool local_cache = false, + int chunk_size = -1, int retries = 0); bool load(const std::byte *data, size_t size, bool no_hw_decoder = false, std::atomic *abort = nullptr); - bool get(int idx, uint8_t *rgb, uint8_t *yuv); - int getRGBSize() const { return aligned_width * aligned_height * 3; } + bool get(int idx, uint8_t *yuv); int getYUVSize() const { return width * height * 3 / 2; } size_t getFrameCount() const { return packets.size(); } bool valid() const { return valid_; } @@ -32,9 +32,9 @@ public: private: bool initHardwareDecoder(AVHWDeviceType hw_device_type); - bool decode(int idx, uint8_t *rgb, uint8_t *yuv); + bool decode(int idx, uint8_t *yuv); AVFrame * decodeFrame(AVPacket *pkt); - bool copyBuffers(AVFrame *f, uint8_t *rgb, uint8_t *yuv); + bool copyBuffers(AVFrame *f, uint8_t *yuv); std::vector packets; std::unique_ptrav_frame_, hw_frame; diff --git a/selfdrive/ui/replay/replay.cc b/selfdrive/ui/replay/replay.cc index 1b1e91a4a..1c8de6bf9 100644 --- a/selfdrive/ui/replay/replay.cc +++ b/selfdrive/ui/replay/replay.cc @@ -289,7 +289,7 @@ void Replay::startStream(const Segment *cur_segment) { camera_size[type] = {fr->width, fr->height}; } } - camera_server_ = std::make_unique(camera_size, true); + camera_server_ = std::make_unique(camera_size); } // start stream thread diff --git a/selfdrive/ui/replay/tests/test_replay.cc b/selfdrive/ui/replay/tests/test_replay.cc index 3cafa9b53..e80c52607 100644 --- a/selfdrive/ui/replay/tests/test_replay.cc +++ b/selfdrive/ui/replay/tests/test_replay.cc @@ -102,11 +102,10 @@ void read_segment(int n, const SegmentFile &segment_file, uint32_t flags) { if (cam == RoadCam || cam == WideRoadCam) { REQUIRE(fr->getFrameCount() == 1200); } - std::unique_ptr rgb_buf = std::make_unique(fr->getRGBSize()); std::unique_ptr yuv_buf = std::make_unique(fr->getYUVSize()); // sequence get 100 frames for (int i = 0; i < 100; ++i) { - REQUIRE(fr->get(i, rgb_buf.get(), yuv_buf.get())); + REQUIRE(fr->get(i, yuv_buf.get())); } } diff --git a/tools/sim/bridge.py b/tools/sim/bridge.py index 8729bd04d..5d40c8ce9 100755 --- a/tools/sim/bridge.py +++ b/tools/sim/bridge.py @@ -70,11 +70,7 @@ class Camerad: self.frame_wide_id = 0 self.vipc_server = VisionIpcServer("camerad") - # TODO: remove RGB buffers once the last RGB vipc subscriber is removed - self.vipc_server.create_buffers(VisionStreamType.VISION_STREAM_RGB_ROAD, 4, True, W, H) self.vipc_server.create_buffers(VisionStreamType.VISION_STREAM_ROAD, 5, False, W, H) - - self.vipc_server.create_buffers(VisionStreamType.VISION_STREAM_RGB_WIDE_ROAD, 4, True, W, H) self.vipc_server.create_buffers(VisionStreamType.VISION_STREAM_WIDE_ROAD, 5, False, W, H) self.vipc_server.start_listener() @@ -92,16 +88,14 @@ class Camerad: self.Hdiv4 = H // 4 if (H % 4 == 0) else (H + (4 - H % 4)) // 4 def cam_callback_road(self, image): - self._cam_callback(image, self.frame_road_id, 'roadCameraState', - VisionStreamType.VISION_STREAM_RGB_ROAD, VisionStreamType.VISION_STREAM_ROAD) + self._cam_callback(image, self.frame_road_id, 'roadCameraState', VisionStreamType.VISION_STREAM_ROAD) self.frame_road_id += 1 def cam_callback_wide_road(self, image): - self._cam_callback(image, self.frame_wide_id, 'wideRoadCameraState', - VisionStreamType.VISION_STREAM_RGB_WIDE_ROAD, VisionStreamType.VISION_STREAM_WIDE_ROAD) + self._cam_callback(image, self.frame_wide_id, 'wideRoadCameraState', VisionStreamType.VISION_STREAM_WIDE_ROAD) self.frame_wide_id += 1 - def _cam_callback(self, image, frame_id, pub_type, rgb_type, yuv_type): + def _cam_callback(self, image, frame_id, pub_type, yuv_type): img = np.frombuffer(image.raw_data, dtype=np.dtype("uint8")) img = np.reshape(img, (H, W, 4)) img = img[:, :, [0, 1, 2]].copy() @@ -114,8 +108,6 @@ class Camerad: yuv = np.resize(yuv_cl.get(), rgb.size // 2) eof = int(frame_id * 0.05 * 1e9) - # TODO: remove RGB send once the last RGB vipc subscriber is removed - self.vipc_server.send(rgb_type, img.tobytes(), frame_id, eof, eof) self.vipc_server.send(yuv_type, yuv.data.tobytes(), frame_id, eof, eof) dat = messaging.new_message(pub_type)