sadmen
/
dragonpilot
mirror of https://github.com/dragonpilot-community/dragonpilot.git
1
0
Fork 0
Code Issues 64 Packages Projects Releases Wiki Activity

replay: cleanup unused RGB buffers (#24537)

Browse Source 
* replay: cleanup unused RGB buffers

* update replay cam

* little more
pull/214/head
Adeeb Shihadeh 4 years ago committed by GitHub
parent d6c6275163
commit ab75c53792
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 39 additions and 66 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 5
      selfdrive/camerad/cameras/camera_replay.cc
  2. 2
      selfdrive/test/process_replay/regen.py
  3. 21
      selfdrive/ui/replay/camera.cc
  4. 14
      selfdrive/ui/replay/camera.h
  5. 20
      selfdrive/ui/replay/framereader.cc
  6. 10
      selfdrive/ui/replay/framereader.h
  7. 2
      selfdrive/ui/replay/replay.cc
  8. 3
      selfdrive/ui/replay/tests/test_replay.cc
  9. 14
      tools/sim/bridge.py

5
selfdrive/camerad/cameras/camera_replay.cc
Unescape View File

@ -47,17 +47,16 @@ void camera_close(CameraState *s) {
void run_camera(CameraState *s) { void run_camera(CameraState *s) {
uint32_t stream_frame_id = 0, frame_id = 0; uint32_t stream_frame_id = 0, frame_id = 0;
size_t buf_idx = 0; size_t buf_idx = 0;
std::unique_ptr<uint8_t[]> rgb_buf = std::make_unique<uint8_t[]>(s->frame->getRGBSize());
std::unique_ptr<uint8_t[]> yuv_buf = std::make_unique<uint8_t[]>(s->frame->getYUVSize()); std::unique_ptr<uint8_t[]> yuv_buf = std::make_unique<uint8_t[]>(s->frame->getYUVSize());
while (!do_exit) { while (!do_exit) {
if (stream_frame_id == s->frame->getFrameCount()) { if (stream_frame_id == s->frame->getFrameCount()) {
// loop stream // loop stream
stream_frame_id = 0; stream_frame_id = 0;
} }
if (s->frame->get(stream_frame_id++, rgb_buf.get(), yuv_buf.get())) { if (s->frame->get(stream_frame_id++, yuv_buf.get())) {
s->buf.camera_bufs_metadata[buf_idx] = {.frame_id = frame_id}; s->buf.camera_bufs_metadata[buf_idx] = {.frame_id = frame_id};
auto &buf = s->buf.camera_bufs[buf_idx]; auto &buf = s->buf.camera_bufs[buf_idx];
CL_CHECK(clEnqueueWriteBuffer(buf.copy_q, buf.buf_cl, CL_TRUE, 0, s->frame->getRGBSize(), rgb_buf.get(), 0, NULL, NULL)); CL_CHECK(clEnqueueWriteBuffer(buf.copy_q, buf.buf_cl, CL_TRUE, 0, s->frame->getYUVSize(), yuv_buf.get(), 0, NULL, NULL));
s->buf.queue(buf_idx); s->buf.queue(buf_idx);
++frame_id; ++frame_id;
buf_idx = (buf_idx + 1) % FRAME_BUF_COUNT; buf_idx = (buf_idx + 1) % FRAME_BUF_COUNT;

2
selfdrive/test/process_replay/regen.py
Unescape View File

@ -165,8 +165,6 @@ def replay_cameras(lr, frs):
p.append(multiprocessing.Process(target=replay_camera, p.append(multiprocessing.Process(target=replay_camera,
args=(s, stream, dt, vs, frames, size, use_extra_client))) args=(s, stream, dt, vs, frames, size, use_extra_client)))
# hack to make UI work
vs.create_buffers(VisionStreamType.VISION_STREAM_RGB_ROAD, 4, True, eon_f_frame_size[0], eon_f_frame_size[1])
vs.start_listener() vs.start_listener()
return vs, p return vs, p

21
selfdrive/ui/replay/camera.cc
Unescape View File

@ -3,7 +3,7 @@
#include <cassert> #include <cassert>
CameraServer::CameraServer(std::pair<int, int> camera_size[MAX_CAMERAS], bool send_yuv) : send_yuv(send_yuv) { CameraServer::CameraServer(std::pair<int, int> camera_size[MAX_CAMERAS]) {
for (int i = 0; i < MAX_CAMERAS; ++i) { for (int i = 0; i < MAX_CAMERAS; ++i) {
std::tie(cameras_[i].width, cameras_[i].height) = camera_size[i]; std::tie(cameras_[i].width, cameras_[i].height) = camera_size[i];
} }
@ -25,10 +25,7 @@ void CameraServer::startVipcServer() {
for (auto &cam : cameras_) { for (auto &cam : cameras_) {
if (cam.width > 0 && cam.height > 0) { if (cam.width > 0 && cam.height > 0) {
rInfo("camera[%d] frame size %dx%d", cam.type, cam.width, cam.height); rInfo("camera[%d] frame size %dx%d", cam.type, cam.width, cam.height);
vipc_server_->create_buffers(cam.rgb_type, UI_BUF_COUNT, true, cam.width, cam.height); vipc_server_->create_buffers(cam.stream_type, YUV_BUFFER_COUNT, false, cam.width, cam.height);
if (send_yuv) {
vipc_server_->create_buffers(cam.yuv_type, YUV_BUFFER_COUNT, false, cam.width, cam.height);
}
if (!cam.thread.joinable()) { if (!cam.thread.joinable()) {
cam.thread = std::thread(&CameraServer::cameraThread, this, std::ref(cam)); cam.thread = std::thread(&CameraServer::cameraThread, this, std::ref(cam));
} }
@ -39,10 +36,9 @@ void CameraServer::startVipcServer() {
void CameraServer::cameraThread(Camera &cam) { void CameraServer::cameraThread(Camera &cam) {
auto read_frame = [&](FrameReader *fr, int frame_id) { auto read_frame = [&](FrameReader *fr, int frame_id) {
VisionBuf *rgb_buf = vipc_server_->get_buffer(cam.rgb_type); VisionBuf *yuv_buf = vipc_server_->get_buffer(cam.stream_type);
VisionBuf *yuv_buf = send_yuv ? vipc_server_->get_buffer(cam.yuv_type) : nullptr; bool ret = fr->get(frame_id, yuv_buf ? (uint8_t *)yuv_buf->addr : nullptr);
bool ret = fr->get(frame_id, (uint8_t *)rgb_buf->addr, yuv_buf ? (uint8_t *)yuv_buf->addr : nullptr); return ret ? yuv_buf : nullptr;
return ret ? std::pair{rgb_buf, yuv_buf} : std::pair{nullptr, nullptr};
}; };
while (true) { while (true) {
@ -51,15 +47,14 @@ void CameraServer::cameraThread(Camera &cam) {
const int id = eidx.getSegmentId(); const int id = eidx.getSegmentId();
bool prefetched = (id == cam.cached_id && eidx.getSegmentNum() == cam.cached_seg); bool prefetched = (id == cam.cached_id && eidx.getSegmentNum() == cam.cached_seg);
auto [rgb, yuv] = prefetched ? cam.cached_buf : read_frame(fr, id); auto yuv = prefetched ? cam.cached_buf : read_frame(fr, id);
if (rgb || yuv) { if (yuv) {
VisionIpcBufExtra extra = { VisionIpcBufExtra extra = {
.frame_id = eidx.getFrameId(), .frame_id = eidx.getFrameId(),
.timestamp_sof = eidx.getTimestampSof(), .timestamp_sof = eidx.getTimestampSof(),
.timestamp_eof = eidx.getTimestampEof(), .timestamp_eof = eidx.getTimestampEof(),
}; };
if (rgb) vipc_server_->send(rgb, &extra, false); vipc_server_->send(yuv, &extra, false);
if (yuv) vipc_server_->send(yuv, &extra, false);
} else { } else {
rError("camera[%d] failed to get frame:", cam.type, eidx.getSegmentId()); rError("camera[%d] failed to get frame:", cam.type, eidx.getSegmentId());
} }

14
selfdrive/ui/replay/camera.h
Unescape View File

@ -8,7 +8,7 @@
class CameraServer { class CameraServer {
public: public:
CameraServer(std::pair<int, int> camera_size[MAX_CAMERAS] = nullptr, bool send_yuv = false); CameraServer(std::pair<int, int> camera_size[MAX_CAMERAS] = nullptr);
~CameraServer(); ~CameraServer();
void pushFrame(CameraType type, FrameReader* fr, const cereal::EncodeIndex::Reader& eidx); void pushFrame(CameraType type, FrameReader* fr, const cereal::EncodeIndex::Reader& eidx);
void waitForSent(); void waitForSent();
@ -16,25 +16,23 @@ public:
protected: protected:
struct Camera { struct Camera {
CameraType type; CameraType type;
VisionStreamType rgb_type; VisionStreamType stream_type;
VisionStreamType yuv_type;
int width; int width;
int height; int height;
std::thread thread; std::thread thread;
SafeQueue<std::pair<FrameReader*, const cereal::EncodeIndex::Reader>> queue; SafeQueue<std::pair<FrameReader*, const cereal::EncodeIndex::Reader>> queue;
int cached_id = -1; int cached_id = -1;
int cached_seg = -1; int cached_seg = -1;
std::pair<VisionBuf *, VisionBuf*> cached_buf; VisionBuf * cached_buf;
}; };
void startVipcServer(); void startVipcServer();
void cameraThread(Camera &cam); void cameraThread(Camera &cam);
Camera cameras_[MAX_CAMERAS] = { Camera cameras_[MAX_CAMERAS] = {
{.type = RoadCam, .rgb_type = VISION_STREAM_RGB_ROAD, .yuv_type = VISION_STREAM_ROAD}, {.type = RoadCam, .stream_type = VISION_STREAM_ROAD},
{.type = DriverCam, .rgb_type = VISION_STREAM_RGB_DRIVER, .yuv_type = VISION_STREAM_DRIVER}, {.type = DriverCam, .stream_type = VISION_STREAM_DRIVER},
{.type = WideRoadCam, .rgb_type = VISION_STREAM_RGB_WIDE_ROAD, .yuv_type = VISION_STREAM_WIDE_ROAD}, {.type = WideRoadCam, .stream_type = VISION_STREAM_WIDE_ROAD},
}; };
std::atomic<int> publishing_ = 0; std::atomic<int> publishing_ = 0;
std::unique_ptr<VisionIpcServer> vipc_server_; std::unique_ptr<VisionIpcServer> vipc_server_;
bool send_yuv;
}; };

20
selfdrive/ui/replay/framereader.cc
Unescape View File

@ -170,15 +170,15 @@ bool FrameReader::initHardwareDecoder(AVHWDeviceType hw_device_type) {
return true; return true;
} }
bool FrameReader::get(int idx, uint8_t *rgb, uint8_t *yuv) { bool FrameReader::get(int idx, uint8_t *yuv) {
assert(rgb || yuv); assert(yuv != nullptr);
if (!valid_ || idx < 0 || idx >= packets.size()) { if (!valid_ || idx < 0 || idx >= packets.size()) {
return false; return false;
} }
return decode(idx, rgb, yuv); return decode(idx, yuv);
} }
bool FrameReader::decode(int idx, uint8_t *rgb, uint8_t *yuv) { bool FrameReader::decode(int idx, uint8_t *yuv) {
int from_idx = idx; int from_idx = idx;
if (idx != prev_idx + 1 && key_frames_count_ > 1) { if (idx != prev_idx + 1 && key_frames_count_ > 1) {
// seeking to the nearest key frame // seeking to the nearest key frame
@ -194,7 +194,7 @@ bool FrameReader::decode(int idx, uint8_t *rgb, uint8_t *yuv) {
for (int i = from_idx; i <= idx; ++i) { for (int i = from_idx; i <= idx; ++i) {
AVFrame *f = decodeFrame(packets[i]); AVFrame *f = decodeFrame(packets[i]);
if (f && i == idx) { if (f && i == idx) {
return copyBuffers(f, rgb, yuv); return copyBuffers(f, yuv);
} }
} }
return false; return false;
@ -226,17 +226,14 @@ AVFrame *FrameReader::decodeFrame(AVPacket *pkt) {
} }
} }
bool FrameReader::copyBuffers(AVFrame *f, uint8_t *rgb, uint8_t *yuv) { bool FrameReader::copyBuffers(AVFrame *f, uint8_t *yuv) {
if (hw_pix_fmt == HW_PIX_FMT) { if (hw_pix_fmt == HW_PIX_FMT) {
uint8_t *y = yuv ? yuv : nv12toyuv_buffer.data(); uint8_t *y = yuv ? yuv : nv12toyuv_buffer.data();
uint8_t *u = y + width * height; uint8_t *u = y + width * height;
uint8_t *v = u + (width / 2) * (height / 2); uint8_t *v = u + (width / 2) * (height / 2);
libyuv::NV12ToI420(f->data[0], f->linesize[0], f->data[1], f->linesize[1], libyuv::NV12ToI420(f->data[0], f->linesize[0], f->data[1], f->linesize[1],
y, width, u, width / 2, v, width / 2, width, height); y, width, u, width / 2, v, width / 2, width, height);
libyuv::I420ToRGB24(y, width, u, width / 2, v, width / 2,
rgb, aligned_width * 3, width, height);
} else { } else {
if (yuv) {
uint8_t *u = yuv + width * height; uint8_t *u = yuv + width * height;
uint8_t *v = u + (width / 2) * (height / 2); uint8_t *v = u + (width / 2) * (height / 2);
libyuv::I420Copy(f->data[0], f->linesize[0], libyuv::I420Copy(f->data[0], f->linesize[0],
@ -245,10 +242,5 @@ bool FrameReader::copyBuffers(AVFrame *f, uint8_t *rgb, uint8_t *yuv) {
yuv, width, u, width / 2, v, width / 2, yuv, width, u, width / 2, v, width / 2,
width, height); width, height);
} }
libyuv::I420ToRGB24(f->data[0], f->linesize[0],
f->data[1], f->linesize[1],
f->data[2], f->linesize[2],
rgb, aligned_width * 3, width, height);
}
return true; return true;
} }

10
selfdrive/ui/replay/framereader.h
Unescape View File

@ -19,10 +19,10 @@ class FrameReader {
public: public:
FrameReader(); FrameReader();
~FrameReader(); ~FrameReader();
bool load(const std::string &url, bool no_hw_decoder = false, std::atomic<bool> *abort = nullptr, bool local_cache = false, int chunk_size = -1, int retries = 0); bool load(const std::string &url, bool no_hw_decoder = false, std::atomic<bool> *abort = nullptr, bool local_cache = false,
int chunk_size = -1, int retries = 0);
bool load(const std::byte *data, size_t size, bool no_hw_decoder = false, std::atomic<bool> *abort = nullptr); bool load(const std::byte *data, size_t size, bool no_hw_decoder = false, std::atomic<bool> *abort = nullptr);
bool get(int idx, uint8_t *rgb, uint8_t *yuv); bool get(int idx, uint8_t *yuv);
int getRGBSize() const { return aligned_width * aligned_height * 3; }
int getYUVSize() const { return width * height * 3 / 2; } int getYUVSize() const { return width * height * 3 / 2; }
size_t getFrameCount() const { return packets.size(); } size_t getFrameCount() const { return packets.size(); }
bool valid() const { return valid_; } bool valid() const { return valid_; }
@ -32,9 +32,9 @@ public:
private: private:
bool initHardwareDecoder(AVHWDeviceType hw_device_type); bool initHardwareDecoder(AVHWDeviceType hw_device_type);
bool decode(int idx, uint8_t *rgb, uint8_t *yuv); bool decode(int idx, uint8_t *yuv);
AVFrame * decodeFrame(AVPacket *pkt); AVFrame * decodeFrame(AVPacket *pkt);
bool copyBuffers(AVFrame *f, uint8_t *rgb, uint8_t *yuv); bool copyBuffers(AVFrame *f, uint8_t *yuv);
std::vector<AVPacket*> packets; std::vector<AVPacket*> packets;
std::unique_ptr<AVFrame, AVFrameDeleter>av_frame_, hw_frame; std::unique_ptr<AVFrame, AVFrameDeleter>av_frame_, hw_frame;

2
selfdrive/ui/replay/replay.cc
Unescape View File

@ -289,7 +289,7 @@ void Replay::startStream(const Segment *cur_segment) {
camera_size[type] = {fr->width, fr->height}; camera_size[type] = {fr->width, fr->height};
} }
} }
camera_server_ = std::make_unique<CameraServer>(camera_size, true); camera_server_ = std::make_unique<CameraServer>(camera_size);
} }
// start stream thread // start stream thread

3
selfdrive/ui/replay/tests/test_replay.cc
Unescape View File

@ -102,11 +102,10 @@ void read_segment(int n, const SegmentFile &segment_file, uint32_t flags) {
if (cam == RoadCam || cam == WideRoadCam) { if (cam == RoadCam || cam == WideRoadCam) {
REQUIRE(fr->getFrameCount() == 1200); REQUIRE(fr->getFrameCount() == 1200);
} }
std::unique_ptr<uint8_t[]> rgb_buf = std::make_unique<uint8_t[]>(fr->getRGBSize());
std::unique_ptr<uint8_t[]> yuv_buf = std::make_unique<uint8_t[]>(fr->getYUVSize()); std::unique_ptr<uint8_t[]> yuv_buf = std::make_unique<uint8_t[]>(fr->getYUVSize());
// sequence get 100 frames // sequence get 100 frames
for (int i = 0; i < 100; ++i) { for (int i = 0; i < 100; ++i) {
REQUIRE(fr->get(i, rgb_buf.get(), yuv_buf.get())); REQUIRE(fr->get(i, yuv_buf.get()));
} }
} }

14
tools/sim/bridge.py
Unescape View File

@ -70,11 +70,7 @@ class Camerad:
self.frame_wide_id = 0 self.frame_wide_id = 0
self.vipc_server = VisionIpcServer("camerad") self.vipc_server = VisionIpcServer("camerad")
# TODO: remove RGB buffers once the last RGB vipc subscriber is removed
self.vipc_server.create_buffers(VisionStreamType.VISION_STREAM_RGB_ROAD, 4, True, W, H)
self.vipc_server.create_buffers(VisionStreamType.VISION_STREAM_ROAD, 5, False, W, H) self.vipc_server.create_buffers(VisionStreamType.VISION_STREAM_ROAD, 5, False, W, H)
self.vipc_server.create_buffers(VisionStreamType.VISION_STREAM_RGB_WIDE_ROAD, 4, True, W, H)
self.vipc_server.create_buffers(VisionStreamType.VISION_STREAM_WIDE_ROAD, 5, False, W, H) self.vipc_server.create_buffers(VisionStreamType.VISION_STREAM_WIDE_ROAD, 5, False, W, H)
self.vipc_server.start_listener() self.vipc_server.start_listener()
@ -92,16 +88,14 @@ class Camerad:
self.Hdiv4 = H // 4 if (H % 4 == 0) else (H + (4 - H % 4)) // 4 self.Hdiv4 = H // 4 if (H % 4 == 0) else (H + (4 - H % 4)) // 4
def cam_callback_road(self, image): def cam_callback_road(self, image):
self._cam_callback(image, self.frame_road_id, 'roadCameraState', self._cam_callback(image, self.frame_road_id, 'roadCameraState', VisionStreamType.VISION_STREAM_ROAD)
VisionStreamType.VISION_STREAM_RGB_ROAD, VisionStreamType.VISION_STREAM_ROAD)
self.frame_road_id += 1 self.frame_road_id += 1
def cam_callback_wide_road(self, image): def cam_callback_wide_road(self, image):
self._cam_callback(image, self.frame_wide_id, 'wideRoadCameraState', self._cam_callback(image, self.frame_wide_id, 'wideRoadCameraState', VisionStreamType.VISION_STREAM_WIDE_ROAD)
VisionStreamType.VISION_STREAM_RGB_WIDE_ROAD, VisionStreamType.VISION_STREAM_WIDE_ROAD)
self.frame_wide_id += 1 self.frame_wide_id += 1
def _cam_callback(self, image, frame_id, pub_type, rgb_type, yuv_type): def _cam_callback(self, image, frame_id, pub_type, yuv_type):
img = np.frombuffer(image.raw_data, dtype=np.dtype("uint8")) img = np.frombuffer(image.raw_data, dtype=np.dtype("uint8"))
img = np.reshape(img, (H, W, 4)) img = np.reshape(img, (H, W, 4))
img = img[:, :, [0, 1, 2]].copy() img = img[:, :, [0, 1, 2]].copy()
@ -114,8 +108,6 @@ class Camerad:
yuv = np.resize(yuv_cl.get(), rgb.size // 2) yuv = np.resize(yuv_cl.get(), rgb.size // 2)
eof = int(frame_id * 0.05 * 1e9) eof = int(frame_id * 0.05 * 1e9)
# TODO: remove RGB send once the last RGB vipc subscriber is removed
self.vipc_server.send(rgb_type, img.tobytes(), frame_id, eof, eof)
self.vipc_server.send(yuv_type, yuv.data.tobytes(), frame_id, eof, eof) self.vipc_server.send(yuv_type, yuv.data.tobytes(), frame_id, eof, eof)
dat = messaging.new_message(pub_type) dat = messaging.new_message(pub_type)

Write Preview
Loading…
Cancel
Save