From 31f9e8af0fbe26ca6a331efede03468ec9639167 Mon Sep 17 00:00:00 2001 From: Joost Wooning Date: Mon, 25 Apr 2022 22:45:55 +0200 Subject: [PATCH] UI: remove memcpy on frame receive (#24318) * UI: remove frame memcpy * fix freezing * fix onroad test old-commit-hash: f84fdc86bbd8223f22158abfa37e368fa18bedf6 --- selfdrive/test/test_onroad.py | 2 +- selfdrive/ui/qt/widgets/cameraview.cc | 77 +++++---------------------- selfdrive/ui/qt/widgets/cameraview.h | 4 +- 3 files changed, 16 insertions(+), 67 deletions(-) diff --git a/selfdrive/test/test_onroad.py b/selfdrive/test/test_onroad.py index 9cca0a35d4..c8c1a34375 100755 --- a/selfdrive/test/test_onroad.py +++ b/selfdrive/test/test_onroad.py @@ -26,7 +26,7 @@ PROCS = { "./camerad": 41.0, "./locationd": 9.1, "selfdrive.controls.plannerd": 11.7, - "./_ui": 33.0, + "./_ui": 18.4, "selfdrive.locationd.paramsd": 9.0, "./_sensord": 6.17, "selfdrive.controls.radard": 4.5, diff --git a/selfdrive/ui/qt/widgets/cameraview.cc b/selfdrive/ui/qt/widgets/cameraview.cc index 6b091bec23..4ef9ef1370 100644 --- a/selfdrive/ui/qt/widgets/cameraview.cc +++ b/selfdrive/ui/qt/widgets/cameraview.cc @@ -102,6 +102,7 @@ CameraViewWidget::CameraViewWidget(std::string stream_name, VisionStreamType typ stream_name(stream_name), stream_type(type), zoomed_view(zoom), QOpenGLWidget(parent) { setAttribute(Qt::WA_OpaquePaintEvent); connect(this, &CameraViewWidget::vipcThreadConnected, this, &CameraViewWidget::vipcConnected, Qt::BlockingQueuedConnection); + connect(this, &CameraViewWidget::vipcThreadFrameReceived, this, &CameraViewWidget::vipcFrameReceived); } CameraViewWidget::~CameraViewWidget() { @@ -162,7 +163,7 @@ void CameraViewWidget::initializeGL() { } void CameraViewWidget::showEvent(QShowEvent *event) { - latest_texture_id = -1; + latest_frame = nullptr; if (!vipc_thread) { vipc_thread = new QThread(); connect(vipc_thread, &QThread::started, [=]() { vipcThread(); }); @@ -215,7 +216,7 @@ void CameraViewWidget::paintGL() { std::lock_guard lk(lock); - if (latest_texture_id == -1) return; + if (latest_frame == nullptr) return; glViewport(0, 0, width(), height()); // sync with the PBO @@ -226,9 +227,13 @@ void CameraViewWidget::paintGL() { glBindVertexArray(frame_vao); glUseProgram(program->programId()); + uint8_t *address[3] = {latest_frame->y, latest_frame->u, latest_frame->v}; for (int i = 0; i < 3; ++i) { glActiveTexture(GL_TEXTURE0 + i); glBindTexture(GL_TEXTURE_2D, textures[i]); + int width = i == 0 ? stream_width : stream_width / 2; + int height = i == 0 ? stream_height : stream_height / 2; + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height, GL_LUMINANCE, GL_UNSIGNED_BYTE, address[i]); assert(glGetError() == GL_NO_ERROR); } @@ -244,7 +249,7 @@ void CameraViewWidget::paintGL() { void CameraViewWidget::vipcConnected(VisionIpcClient *vipc_client) { makeCurrent(); - latest_texture_id = -1; + latest_frame = nullptr; stream_width = vipc_client->buffers[0].width; stream_height = vipc_client->buffers[0].height; @@ -264,27 +269,15 @@ void CameraViewWidget::vipcConnected(VisionIpcClient *vipc_client) { updateFrameMat(width(), height()); } +void CameraViewWidget::vipcFrameReceived(VisionBuf *buf) { + latest_frame = buf; + update(); +} + void CameraViewWidget::vipcThread() { VisionStreamType cur_stream_type = stream_type; std::unique_ptr vipc_client; - std::unique_ptr ctx; - std::unique_ptr surface; - std::unique_ptr gl_buffer; - - ctx = std::make_unique(); - ctx->setFormat(context()->format()); - ctx->setShareContext(context()); - ctx->create(); - assert(ctx->isValid()); - - surface = std::make_unique(); - surface->setFormat(ctx->format()); - surface->create(); - ctx->makeCurrent(surface.get()); - assert(QOpenGLContext::currentContext() == ctx.get()); - initializeOpenGLFunctions(); - while (!QThread::currentThread()->isInterruptionRequested()) { if (!vipc_client || cur_stream_type != stream_type) { cur_stream_type = stream_type; @@ -296,54 +289,10 @@ void CameraViewWidget::vipcThread() { QThread::msleep(100); continue; } - - gl_buffer.reset(new QOpenGLBuffer(QOpenGLBuffer::PixelUnpackBuffer)); - gl_buffer->create(); - gl_buffer->bind(); - gl_buffer->setUsagePattern(QOpenGLBuffer::StreamDraw); - gl_buffer->allocate(vipc_client->buffers[0].len); - emit vipcThreadConnected(vipc_client.get()); } if (VisionBuf *buf = vipc_client->recv(nullptr, 1000)) { - { - std::lock_guard lk(lock); - - for (int i = 0; i < 3; i++) { - void *texture_buffer = gl_buffer->map(QOpenGLBuffer::WriteOnly); - - if (texture_buffer == nullptr) { - LOGE("gl_buffer->map returned nullptr"); - continue; - } - - int width = i == 0 ? stream_width : stream_width / 2; - int height = i == 0 ? stream_height : stream_height / 2; - uint8_t* tex_buf[] = {buf->y, buf->u, buf->v}; - memcpy(texture_buffer, tex_buf[i], width*height); - gl_buffer->unmap(); - - // copy pixels from PBO to texture object - glBindTexture(GL_TEXTURE_2D, textures[i]); - glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height, GL_LUMINANCE, GL_UNSIGNED_BYTE, 0); - } - - glBindTexture(GL_TEXTURE_2D, 0); - assert(glGetError() == GL_NO_ERROR); - - wait_fence.reset(new WaitFence()); - - // Ensure the fence is in the GPU command queue, or waiting on it might block - // https://www.khronos.org/opengl/wiki/Sync_Object#Flushing_and_contexts - glFlush(); - - latest_texture_id = buf->idx; - } - // Schedule update. update() will be invoked on the gui thread. - QMetaObject::invokeMethod(this, "update"); - - // TODO: remove later, it's only connected by DriverView. emit vipcThreadFrameReceived(buf); } } diff --git a/selfdrive/ui/qt/widgets/cameraview.h b/selfdrive/ui/qt/widgets/cameraview.h index a46b9c9e3d..1f19ea157b 100644 --- a/selfdrive/ui/qt/widgets/cameraview.h +++ b/selfdrive/ui/qt/widgets/cameraview.h @@ -8,7 +8,6 @@ #include #include "cereal/visionipc/visionipc_client.h" #include "selfdrive/camerad/cameras/camera_common.h" -#include "selfdrive/common/visionimg.h" #include "selfdrive/ui/ui.h" class CameraViewWidget : public QOpenGLWidget, protected QOpenGLFunctions { @@ -45,7 +44,7 @@ protected: bool zoomed_view; std::mutex lock; - int latest_texture_id = -1; + VisionBuf *latest_frame = nullptr; GLuint frame_vao, frame_vbo, frame_ibo; mat4 frame_mat; std::unique_ptr wait_fence; @@ -62,4 +61,5 @@ protected: protected slots: void vipcConnected(VisionIpcClient *vipc_client); + void vipcFrameReceived(VisionBuf *vipc_client); };