sadmen
/
jyoung
mirror of https://github.com/jyoung8607/openpilot.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

replay: cleanup unused RGB buffers (#24537)

Browse Source 
* replay: cleanup unused RGB buffers

* update replay cam

* little more
old-commit-hash: ab75c53792
vw-mqb-aeb
Adeeb Shihadeh 3 years ago committed by GitHub
parent 874ba219dc
commit 29a516bed8
9 changed files with 39 additions and 66 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 5
      selfdrive/camerad/cameras/camera_replay.cc
  2. 2
      selfdrive/test/process_replay/regen.py
  3. 21
      selfdrive/ui/replay/camera.cc
  4. 14
      selfdrive/ui/replay/camera.h
  5. 20
      selfdrive/ui/replay/framereader.cc
  6. 10
      selfdrive/ui/replay/framereader.h
  7. 2
      selfdrive/ui/replay/replay.cc
  8. 3
      selfdrive/ui/replay/tests/test_replay.cc
  9. 14
      tools/sim/bridge.py

5
selfdrive/camerad/cameras/camera_replay.cc
Unescape View File

@ -47,17 +47,16 @@ void camera_close(CameraState *s) {
void run_camera(CameraState *s) {
uint32_t stream_frame_id = 0, frame_id = 0;
size_t buf_idx = 0;
std::unique_ptr<uint8_t[]> rgb_buf = std::make_unique<uint8_t[]>(s->frame->getRGBSize());
std::unique_ptr<uint8_t[]> yuv_buf = std::make_unique<uint8_t[]>(s->frame->getYUVSize());
while (!do_exit) {
if (stream_frame_id == s->frame->getFrameCount()) {
// loop stream
stream_frame_id = 0;
}
if (s->frame->get(stream_frame_id++, rgb_buf.get(), yuv_buf.get())) {
if (s->frame->get(stream_frame_id++, yuv_buf.get())) {
s->buf.camera_bufs_metadata[buf_idx] = {.frame_id = frame_id};
auto &buf = s->buf.camera_bufs[buf_idx];
CL_CHECK(clEnqueueWriteBuffer(buf.copy_q, buf.buf_cl, CL_TRUE, 0, s->frame->getRGBSize(), rgb_buf.get(), 0, NULL, NULL));
CL_CHECK(clEnqueueWriteBuffer(buf.copy_q, buf.buf_cl, CL_TRUE, 0, s->frame->getYUVSize(), yuv_buf.get(), 0, NULL, NULL));
s->buf.queue(buf_idx);
++frame_id;
buf_idx = (buf_idx + 1) % FRAME_BUF_COUNT;

2
selfdrive/test/process_replay/regen.py
Unescape View File

@ -165,8 +165,6 @@ def replay_cameras(lr, frs):
p.append(multiprocessing.Process(target=replay_camera,
args=(s, stream, dt, vs, frames, size, use_extra_client)))
# hack to make UI work
vs.create_buffers(VisionStreamType.VISION_STREAM_RGB_ROAD, 4, True, eon_f_frame_size[0], eon_f_frame_size[1])
vs.start_listener()
return vs, p

21
selfdrive/ui/replay/camera.cc
Unescape View File

@ -3,7 +3,7 @@
#include <cassert>
CameraServer::CameraServer(std::pair<int, int> camera_size[MAX_CAMERAS], bool send_yuv) : send_yuv(send_yuv) {
CameraServer::CameraServer(std::pair<int, int> camera_size[MAX_CAMERAS]) {
for (int i = 0; i < MAX_CAMERAS; ++i) {
std::tie(cameras_[i].width, cameras_[i].height) = camera_size[i];
}
@ -25,10 +25,7 @@ void CameraServer::startVipcServer() {
for (auto &cam : cameras_) {
if (cam.width > 0 && cam.height > 0) {
rInfo("camera[%d] frame size %dx%d", cam.type, cam.width, cam.height);
vipc_server_->create_buffers(cam.rgb_type, UI_BUF_COUNT, true, cam.width, cam.height);
if (send_yuv) {
vipc_server_->create_buffers(cam.yuv_type, YUV_BUFFER_COUNT, false, cam.width, cam.height);
}
vipc_server_->create_buffers(cam.stream_type, YUV_BUFFER_COUNT, false, cam.width, cam.height);
if (!cam.thread.joinable()) {
cam.thread = std::thread(&CameraServer::cameraThread, this, std::ref(cam));
}
@ -39,10 +36,9 @@ void CameraServer::startVipcServer() {
void CameraServer::cameraThread(Camera &cam) {
auto read_frame = [&](FrameReader *fr, int frame_id) {
VisionBuf *rgb_buf = vipc_server_->get_buffer(cam.rgb_type);
VisionBuf *yuv_buf = send_yuv ? vipc_server_->get_buffer(cam.yuv_type) : nullptr;
bool ret = fr->get(frame_id, (uint8_t *)rgb_buf->addr, yuv_buf ? (uint8_t *)yuv_buf->addr : nullptr);
return ret ? std::pair{rgb_buf, yuv_buf} : std::pair{nullptr, nullptr};
VisionBuf *yuv_buf = vipc_server_->get_buffer(cam.stream_type);
bool ret = fr->get(frame_id, yuv_buf ? (uint8_t *)yuv_buf->addr : nullptr);
return ret ? yuv_buf : nullptr;
};
while (true) {
@ -51,15 +47,14 @@ void CameraServer::cameraThread(Camera &cam) {
const int id = eidx.getSegmentId();
bool prefetched = (id == cam.cached_id && eidx.getSegmentNum() == cam.cached_seg);
auto [rgb, yuv] = prefetched ? cam.cached_buf : read_frame(fr, id);
if (rgb || yuv) {
auto yuv = prefetched ? cam.cached_buf : read_frame(fr, id);
if (yuv) {
VisionIpcBufExtra extra = {
.frame_id = eidx.getFrameId(),
.timestamp_sof = eidx.getTimestampSof(),
.timestamp_eof = eidx.getTimestampEof(),
};
if (rgb) vipc_server_->send(rgb, &extra, false);
if (yuv) vipc_server_->send(yuv, &extra, false);
vipc_server_->send(yuv, &extra, false);
} else {
rError("camera[%d] failed to get frame:", cam.type, eidx.getSegmentId());
}

14
selfdrive/ui/replay/camera.h
Unescape View File

@ -8,7 +8,7 @@
class CameraServer {
public:
CameraServer(std::pair<int, int> camera_size[MAX_CAMERAS] = nullptr, bool send_yuv = false);
CameraServer(std::pair<int, int> camera_size[MAX_CAMERAS] = nullptr);
~CameraServer();
void pushFrame(CameraType type, FrameReader* fr, const cereal::EncodeIndex::Reader& eidx);
void waitForSent();
@ -16,25 +16,23 @@ public:
protected:
struct Camera {
CameraType type;
VisionStreamType rgb_type;
VisionStreamType yuv_type;
VisionStreamType stream_type;
int width;
int height;
std::thread thread;
SafeQueue<std::pair<FrameReader*, const cereal::EncodeIndex::Reader>> queue;
int cached_id = -1;
int cached_seg = -1;
std::pair<VisionBuf *, VisionBuf*> cached_buf;
VisionBuf * cached_buf;
};
void startVipcServer();
void cameraThread(Camera &cam);
Camera cameras_[MAX_CAMERAS] = {
{.type = RoadCam, .rgb_type = VISION_STREAM_RGB_ROAD, .yuv_type = VISION_STREAM_ROAD},
{.type = DriverCam, .rgb_type = VISION_STREAM_RGB_DRIVER, .yuv_type = VISION_STREAM_DRIVER},
{.type = WideRoadCam, .rgb_type = VISION_STREAM_RGB_WIDE_ROAD, .yuv_type = VISION_STREAM_WIDE_ROAD},
{.type = RoadCam, .stream_type = VISION_STREAM_ROAD},
{.type = DriverCam, .stream_type = VISION_STREAM_DRIVER},
{.type = WideRoadCam, .stream_type = VISION_STREAM_WIDE_ROAD},
};
std::atomic<int> publishing_ = 0;
std::unique_ptr<VisionIpcServer> vipc_server_;
bool send_yuv;
};

20
selfdrive/ui/replay/framereader.cc
Unescape View File

@ -170,15 +170,15 @@ bool FrameReader::initHardwareDecoder(AVHWDeviceType hw_device_type) {
return true;
}
bool FrameReader::get(int idx, uint8_t *rgb, uint8_t *yuv) {
assert(rgb || yuv);
bool FrameReader::get(int idx, uint8_t *yuv) {
assert(yuv != nullptr);
if (!valid_ || idx < 0 || idx >= packets.size()) {
return false;
}
return decode(idx, rgb, yuv);
return decode(idx, yuv);
}
bool FrameReader::decode(int idx, uint8_t *rgb, uint8_t *yuv) {
bool FrameReader::decode(int idx, uint8_t *yuv) {
int from_idx = idx;
if (idx != prev_idx + 1 && key_frames_count_ > 1) {
// seeking to the nearest key frame
@ -194,7 +194,7 @@ bool FrameReader::decode(int idx, uint8_t *rgb, uint8_t *yuv) {
for (int i = from_idx; i <= idx; ++i) {
AVFrame *f = decodeFrame(packets[i]);
if (f && i == idx) {
return copyBuffers(f, rgb, yuv);
return copyBuffers(f, yuv);
}
}
return false;
@ -226,17 +226,14 @@ AVFrame *FrameReader::decodeFrame(AVPacket *pkt) {
}
}
bool FrameReader::copyBuffers(AVFrame *f, uint8_t *rgb, uint8_t *yuv) {
bool FrameReader::copyBuffers(AVFrame *f, uint8_t *yuv) {
if (hw_pix_fmt == HW_PIX_FMT) {
uint8_t *y = yuv ? yuv : nv12toyuv_buffer.data();
uint8_t *u = y + width * height;
uint8_t *v = u + (width / 2) * (height / 2);
libyuv::NV12ToI420(f->data[0], f->linesize[0], f->data[1], f->linesize[1],
y, width, u, width / 2, v, width / 2, width, height);
libyuv::I420ToRGB24(y, width, u, width / 2, v, width / 2,
rgb, aligned_width * 3, width, height);
} else {
if (yuv) {
uint8_t *u = yuv + width * height;
uint8_t *v = u + (width / 2) * (height / 2);
libyuv::I420Copy(f->data[0], f->linesize[0],
@ -245,10 +242,5 @@ bool FrameReader::copyBuffers(AVFrame *f, uint8_t *rgb, uint8_t *yuv) {
yuv, width, u, width / 2, v, width / 2,
width, height);
}
libyuv::I420ToRGB24(f->data[0], f->linesize[0],
f->data[1], f->linesize[1],
f->data[2], f->linesize[2],
rgb, aligned_width * 3, width, height);
}
return true;
}

10
selfdrive/ui/replay/framereader.h
Unescape View File

@ -19,10 +19,10 @@ class FrameReader {
public:
FrameReader();
~FrameReader();
bool load(const std::string &url, bool no_hw_decoder = false, std::atomic<bool> *abort = nullptr, bool local_cache = false, int chunk_size = -1, int retries = 0);
bool load(const std::string &url, bool no_hw_decoder = false, std::atomic<bool> *abort = nullptr, bool local_cache = false,
int chunk_size = -1, int retries = 0);
bool load(const std::byte *data, size_t size, bool no_hw_decoder = false, std::atomic<bool> *abort = nullptr);
bool get(int idx, uint8_t *rgb, uint8_t *yuv);
int getRGBSize() const { return aligned_width * aligned_height * 3; }
bool get(int idx, uint8_t *yuv);
int getYUVSize() const { return width * height * 3 / 2; }
size_t getFrameCount() const { return packets.size(); }
bool valid() const { return valid_; }
@ -32,9 +32,9 @@ public:
private:
bool initHardwareDecoder(AVHWDeviceType hw_device_type);
bool decode(int idx, uint8_t *rgb, uint8_t *yuv);
bool decode(int idx, uint8_t *yuv);
AVFrame * decodeFrame(AVPacket *pkt);
bool copyBuffers(AVFrame *f, uint8_t *rgb, uint8_t *yuv);
bool copyBuffers(AVFrame *f, uint8_t *yuv);
std::vector<AVPacket*> packets;
std::unique_ptr<AVFrame, AVFrameDeleter>av_frame_, hw_frame;

2
selfdrive/ui/replay/replay.cc
Unescape View File

@ -289,7 +289,7 @@ void Replay::startStream(const Segment *cur_segment) {
camera_size[type] = {fr->width, fr->height};
}
}
camera_server_ = std::make_unique<CameraServer>(camera_size, true);
camera_server_ = std::make_unique<CameraServer>(camera_size);
}
// start stream thread

3
selfdrive/ui/replay/tests/test_replay.cc
Unescape View File

@ -102,11 +102,10 @@ void read_segment(int n, const SegmentFile &segment_file, uint32_t flags) {
if (cam == RoadCam || cam == WideRoadCam) {
REQUIRE(fr->getFrameCount() == 1200);
}
std::unique_ptr<uint8_t[]> rgb_buf = std::make_unique<uint8_t[]>(fr->getRGBSize());
std::unique_ptr<uint8_t[]> yuv_buf = std::make_unique<uint8_t[]>(fr->getYUVSize());
// sequence get 100 frames
for (int i = 0; i < 100; ++i) {
REQUIRE(fr->get(i, rgb_buf.get(), yuv_buf.get()));
REQUIRE(fr->get(i, yuv_buf.get()));
}
}

14
tools/sim/bridge.py
Unescape View File

@ -70,11 +70,7 @@ class Camerad:
self.frame_wide_id = 0
self.vipc_server = VisionIpcServer("camerad")
# TODO: remove RGB buffers once the last RGB vipc subscriber is removed
self.vipc_server.create_buffers(VisionStreamType.VISION_STREAM_RGB_ROAD, 4, True, W, H)
self.vipc_server.create_buffers(VisionStreamType.VISION_STREAM_ROAD, 5, False, W, H)
self.vipc_server.create_buffers(VisionStreamType.VISION_STREAM_RGB_WIDE_ROAD, 4, True, W, H)
self.vipc_server.create_buffers(VisionStreamType.VISION_STREAM_WIDE_ROAD, 5, False, W, H)
self.vipc_server.start_listener()
@ -92,16 +88,14 @@ class Camerad:
self.Hdiv4 = H // 4 if (H % 4 == 0) else (H + (4 - H % 4)) // 4
def cam_callback_road(self, image):
self._cam_callback(image, self.frame_road_id, 'roadCameraState',
VisionStreamType.VISION_STREAM_RGB_ROAD, VisionStreamType.VISION_STREAM_ROAD)
self._cam_callback(image, self.frame_road_id, 'roadCameraState', VisionStreamType.VISION_STREAM_ROAD)
self.frame_road_id += 1
def cam_callback_wide_road(self, image):
self._cam_callback(image, self.frame_wide_id, 'wideRoadCameraState',
VisionStreamType.VISION_STREAM_RGB_WIDE_ROAD, VisionStreamType.VISION_STREAM_WIDE_ROAD)
self._cam_callback(image, self.frame_wide_id, 'wideRoadCameraState', VisionStreamType.VISION_STREAM_WIDE_ROAD)
self.frame_wide_id += 1
def _cam_callback(self, image, frame_id, pub_type, rgb_type, yuv_type):
def _cam_callback(self, image, frame_id, pub_type, yuv_type):
img = np.frombuffer(image.raw_data, dtype=np.dtype("uint8"))
img = np.reshape(img, (H, W, 4))
img = img[:, :, [0, 1, 2]].copy()
@ -114,8 +108,6 @@ class Camerad:
yuv = np.resize(yuv_cl.get(), rgb.size // 2)
eof = int(frame_id * 0.05 * 1e9)
# TODO: remove RGB send once the last RGB vipc subscriber is removed
self.vipc_server.send(rgb_type, img.tobytes(), frame_id, eof, eof)
self.vipc_server.send(yuv_type, yuv.data.tobytes(), frame_id, eof, eof)
dat = messaging.new_message(pub_type)

Write Preview
Loading…
Cancel
Save