UI: remove RGB buffer usage (#24298)

* ui: use yuv buffer

* cleanup

* fix watch3

* replay always send yuv

* fix drawing of path

* fix onroad test

Co-authored-by: Willem Melching <willem.melching@gmail.com>
pull/24300/head
Joost Wooning 3 years ago committed by GitHub
parent 9be23cbdb9
commit b9907e86b2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      selfdrive/test/test_onroad.py
  2. 2
      selfdrive/ui/qt/offroad/driverview.cc
  3. 4
      selfdrive/ui/qt/onroad.cc
  4. 141
      selfdrive/ui/qt/widgets/cameraview.cc
  5. 6
      selfdrive/ui/qt/widgets/cameraview.h
  6. 1
      selfdrive/ui/replay/main.cc
  7. 2
      selfdrive/ui/replay/replay.cc
  8. 1
      selfdrive/ui/replay/replay.h
  9. 9
      selfdrive/ui/watch3.cc

@ -26,7 +26,7 @@ PROCS = {
"./camerad": 41.0, "./camerad": 41.0,
"./locationd": 9.1, "./locationd": 9.1,
"selfdrive.controls.plannerd": 11.7, "selfdrive.controls.plannerd": 11.7,
"./_ui": 33.0, "./_ui": 18.4,
"selfdrive.locationd.paramsd": 9.0, "selfdrive.locationd.paramsd": 9.0,
"./_sensord": 6.17, "./_sensord": 6.17,
"selfdrive.controls.radard": 4.5, "selfdrive.controls.radard": 4.5,

@ -12,7 +12,7 @@ DriverViewWindow::DriverViewWindow(QWidget* parent) : QWidget(parent) {
layout = new QStackedLayout(this); layout = new QStackedLayout(this);
layout->setStackingMode(QStackedLayout::StackAll); layout->setStackingMode(QStackedLayout::StackAll);
cameraView = new CameraViewWidget("camerad", VISION_STREAM_RGB_DRIVER, true, this); cameraView = new CameraViewWidget("camerad", VISION_STREAM_DRIVER, true, this);
layout->addWidget(cameraView); layout->addWidget(cameraView);
scene = new DriverViewScene(this); scene = new DriverViewScene(this);

@ -20,7 +20,7 @@ OnroadWindow::OnroadWindow(QWidget *parent) : QWidget(parent) {
QStackedLayout *road_view_layout = new QStackedLayout; QStackedLayout *road_view_layout = new QStackedLayout;
road_view_layout->setStackingMode(QStackedLayout::StackAll); road_view_layout->setStackingMode(QStackedLayout::StackAll);
nvg = new NvgWindow(VISION_STREAM_RGB_ROAD, this); nvg = new NvgWindow(VISION_STREAM_ROAD, this);
road_view_layout->addWidget(nvg); road_view_layout->addWidget(nvg);
hud = new OnroadHud(this); hud = new OnroadHud(this);
road_view_layout->addWidget(hud); road_view_layout->addWidget(hud);
@ -97,7 +97,7 @@ void OnroadWindow::offroadTransition(bool offroad) {
// update stream type // update stream type
bool wide_cam = Hardware::TICI() && Params().getBool("EnableWideCamera"); bool wide_cam = Hardware::TICI() && Params().getBool("EnableWideCamera");
nvg->setStreamType(wide_cam ? VISION_STREAM_RGB_WIDE_ROAD : VISION_STREAM_RGB_ROAD); nvg->setStreamType(wide_cam ? VISION_STREAM_WIDE_ROAD : VISION_STREAM_ROAD);
} }
void OnroadWindow::paintEvent(QPaintEvent *event) { void OnroadWindow::paintEvent(QPaintEvent *event) {

@ -17,10 +17,10 @@ const char frame_vertex_shader[] =
#else #else
"#version 300 es\n" "#version 300 es\n"
#endif #endif
"in vec4 aPosition;\n" "layout(location = 0) in vec4 aPosition;\n"
"in vec4 aTexCoord;\n" "layout(location = 1) in vec2 aTexCoord;\n"
"uniform mat4 uTransform;\n" "uniform mat4 uTransform;\n"
"out vec4 vTexCoord;\n" "out vec2 vTexCoord;\n"
"void main() {\n" "void main() {\n"
" gl_Position = uTransform * aPosition;\n" " gl_Position = uTransform * aPosition;\n"
" vTexCoord = aTexCoord;\n" " vTexCoord = aTexCoord;\n"
@ -33,11 +33,19 @@ const char frame_fragment_shader[] =
"#version 300 es\n" "#version 300 es\n"
"precision mediump float;\n" "precision mediump float;\n"
#endif #endif
"uniform sampler2D uTexture;\n" "uniform sampler2D uTextureY;\n"
"in vec4 vTexCoord;\n" "uniform sampler2D uTextureU;\n"
"uniform sampler2D uTextureV;\n"
"in vec2 vTexCoord;\n"
"out vec4 colorOut;\n" "out vec4 colorOut;\n"
"void main() {\n" "void main() {\n"
" colorOut = texture(uTexture, vTexCoord.xy);\n" " float y = texture(uTextureY, vTexCoord).r;\n"
" float u = texture(uTextureU, vTexCoord).r - 0.5;\n"
" float v = texture(uTextureV, vTexCoord).r - 0.5;\n"
" float r = y + 1.402 * v;\n"
" float g = y - 0.344 * u - 0.714 * v;\n"
" float b = y + 1.772 * u;\n"
" colorOut = vec4(r, g, b, 1.0);\n"
"}\n"; "}\n";
const mat4 device_transform = {{ const mat4 device_transform = {{
@ -94,6 +102,10 @@ CameraViewWidget::CameraViewWidget(std::string stream_name, VisionStreamType typ
stream_name(stream_name), stream_type(type), zoomed_view(zoom), QOpenGLWidget(parent) { stream_name(stream_name), stream_type(type), zoomed_view(zoom), QOpenGLWidget(parent) {
setAttribute(Qt::WA_OpaquePaintEvent); setAttribute(Qt::WA_OpaquePaintEvent);
connect(this, &CameraViewWidget::vipcThreadConnected, this, &CameraViewWidget::vipcConnected, Qt::BlockingQueuedConnection); connect(this, &CameraViewWidget::vipcThreadConnected, this, &CameraViewWidget::vipcConnected, Qt::BlockingQueuedConnection);
connect(this, &CameraViewWidget::vipcThreadFrameReceived, [=](VisionBuf *buf) {
latest_frame = buf;
update();
});
} }
CameraViewWidget::~CameraViewWidget() { CameraViewWidget::~CameraViewWidget() {
@ -102,6 +114,7 @@ CameraViewWidget::~CameraViewWidget() {
glDeleteVertexArrays(1, &frame_vao); glDeleteVertexArrays(1, &frame_vao);
glDeleteBuffers(1, &frame_vbo); glDeleteBuffers(1, &frame_vbo);
glDeleteBuffers(1, &frame_ibo); glDeleteBuffers(1, &frame_ibo);
glDeleteBuffers(3, textures);
} }
doneCurrent(); doneCurrent();
} }
@ -119,7 +132,7 @@ void CameraViewWidget::initializeGL() {
GLint frame_pos_loc = program->attributeLocation("aPosition"); GLint frame_pos_loc = program->attributeLocation("aPosition");
GLint frame_texcoord_loc = program->attributeLocation("aTexCoord"); GLint frame_texcoord_loc = program->attributeLocation("aTexCoord");
auto [x1, x2, y1, y2] = stream_type == VISION_STREAM_RGB_DRIVER ? std::tuple(0.f, 1.f, 1.f, 0.f) : std::tuple(1.f, 0.f, 1.f, 0.f); auto [x1, x2, y1, y2] = stream_type == VISION_STREAM_DRIVER ? std::tuple(0.f, 1.f, 1.f, 0.f) : std::tuple(1.f, 0.f, 1.f, 0.f);
const uint8_t frame_indicies[] = {0, 1, 2, 0, 2, 3}; const uint8_t frame_indicies[] = {0, 1, 2, 0, 2, 3};
const float frame_coords[4][4] = { const float frame_coords[4][4] = {
{-1.0, -1.0, x2, y1}, // bl {-1.0, -1.0, x2, y1}, // bl
@ -144,10 +157,16 @@ void CameraViewWidget::initializeGL() {
glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(frame_indicies), frame_indicies, GL_STATIC_DRAW); glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(frame_indicies), frame_indicies, GL_STATIC_DRAW);
glBindBuffer(GL_ARRAY_BUFFER, 0); glBindBuffer(GL_ARRAY_BUFFER, 0);
glBindVertexArray(0); glBindVertexArray(0);
glGenTextures(3, textures);
glUseProgram(program->programId());
glUniform1i(program->uniformLocation("uTextureY"), 0);
glUniform1i(program->uniformLocation("uTextureU"), 1);
glUniform1i(program->uniformLocation("uTextureV"), 2);
} }
void CameraViewWidget::showEvent(QShowEvent *event) { void CameraViewWidget::showEvent(QShowEvent *event) {
latest_texture_id = -1; latest_frame = nullptr;
if (!vipc_thread) { if (!vipc_thread) {
vipc_thread = new QThread(); vipc_thread = new QThread();
connect(vipc_thread, &QThread::started, [=]() { vipcThread(); }); connect(vipc_thread, &QThread::started, [=]() { vipcThread(); });
@ -167,12 +186,12 @@ void CameraViewWidget::hideEvent(QHideEvent *event) {
void CameraViewWidget::updateFrameMat(int w, int h) { void CameraViewWidget::updateFrameMat(int w, int h) {
if (zoomed_view) { if (zoomed_view) {
if (stream_type == VISION_STREAM_RGB_DRIVER) { if (stream_type == VISION_STREAM_DRIVER) {
frame_mat = matmul(device_transform, get_driver_view_transform(w, h, stream_width, stream_height)); frame_mat = matmul(device_transform, get_driver_view_transform(w, h, stream_width, stream_height));
} else { } else {
auto intrinsic_matrix = stream_type == VISION_STREAM_RGB_WIDE_ROAD ? ecam_intrinsic_matrix : fcam_intrinsic_matrix; auto intrinsic_matrix = stream_type == VISION_STREAM_WIDE_ROAD ? ecam_intrinsic_matrix : fcam_intrinsic_matrix;
float zoom = ZOOM / intrinsic_matrix.v[0]; float zoom = ZOOM / intrinsic_matrix.v[0];
if (stream_type == VISION_STREAM_RGB_WIDE_ROAD) { if (stream_type == VISION_STREAM_WIDE_ROAD) {
zoom *= 0.5; zoom *= 0.5;
} }
float zx = zoom * 2 * intrinsic_matrix.v[2] / width(); float zx = zoom * 2 * intrinsic_matrix.v[2] / width();
@ -200,7 +219,7 @@ void CameraViewWidget::paintGL() {
std::lock_guard lk(lock); std::lock_guard lk(lock);
if (latest_texture_id == -1) return; if (latest_frame == nullptr) return;
glViewport(0, 0, width(), height()); glViewport(0, 0, width(), height());
// sync with the PBO // sync with the PBO
@ -209,13 +228,21 @@ void CameraViewWidget::paintGL() {
} }
glBindVertexArray(frame_vao); glBindVertexArray(frame_vao);
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, texture[latest_texture_id]->frame_tex);
glUseProgram(program->programId()); glUseProgram(program->programId());
glUniform1i(program->uniformLocation("uTexture"), 0); uint8_t *address[3] = {latest_frame->y, latest_frame->u, latest_frame->v};
glUniformMatrix4fv(program->uniformLocation("uTransform"), 1, GL_TRUE, frame_mat.v); for (int i = 0; i < 3; ++i) {
glActiveTexture(GL_TEXTURE0 + i);
glBindTexture(GL_TEXTURE_2D, textures[i]);
int width = i == 0 ? stream_width : stream_width / 2;
int height = i == 0 ? stream_height : stream_height / 2;
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height, GL_LUMINANCE, GL_UNSIGNED_BYTE, address[i]);
assert(glGetError() == GL_NO_ERROR);
}
glActiveTexture(GL_TEXTURE0); // qt requires active texture 0
glUniformMatrix4fv(program->uniformLocation("uTransform"), 1, GL_TRUE, frame_mat.v);
assert(glGetError() == GL_NO_ERROR); assert(glGetError() == GL_NO_ERROR);
glEnableVertexAttribArray(0); glEnableVertexAttribArray(0);
glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_BYTE, (const void *)0); glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_BYTE, (const void *)0);
@ -225,22 +252,23 @@ void CameraViewWidget::paintGL() {
void CameraViewWidget::vipcConnected(VisionIpcClient *vipc_client) { void CameraViewWidget::vipcConnected(VisionIpcClient *vipc_client) {
makeCurrent(); makeCurrent();
for (int i = 0; i < vipc_client->num_buffers; i++) { latest_frame = nullptr;
texture[i].reset(new EGLImageTexture(&vipc_client->buffers[i])); stream_width = vipc_client->buffers[0].width;
stream_height = vipc_client->buffers[0].height;
glBindTexture(GL_TEXTURE_2D, texture[i]->frame_tex);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
// BGR glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_R, GL_BLUE); for (int i = 0; i < 3; ++i) {
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_G, GL_GREEN); glBindTexture(GL_TEXTURE_2D, textures[i]);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_B, GL_RED); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
int width = i == 0 ? stream_width : stream_width / 2;
int height = i == 0 ? stream_height : stream_height / 2;
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, nullptr);
assert(glGetError() == GL_NO_ERROR); assert(glGetError() == GL_NO_ERROR);
} }
latest_texture_id = -1;
stream_width = vipc_client->buffers[0].width;
stream_height = vipc_client->buffers[0].height;
updateFrameMat(width(), height()); updateFrameMat(width(), height());
} }
@ -248,23 +276,6 @@ void CameraViewWidget::vipcThread() {
VisionStreamType cur_stream_type = stream_type; VisionStreamType cur_stream_type = stream_type;
std::unique_ptr<VisionIpcClient> vipc_client; std::unique_ptr<VisionIpcClient> vipc_client;
std::unique_ptr<QOpenGLContext> ctx;
std::unique_ptr<QOffscreenSurface> surface;
std::unique_ptr<QOpenGLBuffer> gl_buffer;
ctx = std::make_unique<QOpenGLContext>();
ctx->setFormat(context()->format());
ctx->setShareContext(context());
ctx->create();
assert(ctx->isValid());
surface = std::make_unique<QOffscreenSurface>();
surface->setFormat(ctx->format());
surface->create();
ctx->makeCurrent(surface.get());
assert(QOpenGLContext::currentContext() == ctx.get());
initializeOpenGLFunctions();
while (!QThread::currentThread()->isInterruptionRequested()) { while (!QThread::currentThread()->isInterruptionRequested()) {
if (!vipc_client || cur_stream_type != stream_type) { if (!vipc_client || cur_stream_type != stream_type) {
cur_stream_type = stream_type; cur_stream_type = stream_type;
@ -276,48 +287,10 @@ void CameraViewWidget::vipcThread() {
QThread::msleep(100); QThread::msleep(100);
continue; continue;
} }
gl_buffer.reset(new QOpenGLBuffer(QOpenGLBuffer::PixelUnpackBuffer));
gl_buffer->create();
gl_buffer->bind();
gl_buffer->setUsagePattern(QOpenGLBuffer::StreamDraw);
gl_buffer->allocate(vipc_client->buffers[0].len);
emit vipcThreadConnected(vipc_client.get()); emit vipcThreadConnected(vipc_client.get());
} }
if (VisionBuf *buf = vipc_client->recv(nullptr, 1000)) { if (VisionBuf *buf = vipc_client->recv(nullptr, 1000)) {
{
std::lock_guard lk(lock);
void *texture_buffer = gl_buffer->map(QOpenGLBuffer::WriteOnly);
if (texture_buffer == nullptr) {
LOGE("gl_buffer->map returned nullptr");
continue;
}
memcpy(texture_buffer, buf->addr, buf->len);
gl_buffer->unmap();
// copy pixels from PBO to texture object
glBindTexture(GL_TEXTURE_2D, texture[buf->idx]->frame_tex);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, buf->width, buf->height, GL_RGB, GL_UNSIGNED_BYTE, 0);
glBindTexture(GL_TEXTURE_2D, 0);
assert(glGetError() == GL_NO_ERROR);
wait_fence.reset(new WaitFence());
// Ensure the fence is in the GPU command queue, or waiting on it might block
// https://www.khronos.org/opengl/wiki/Sync_Object#Flushing_and_contexts
glFlush();
latest_texture_id = buf->idx;
}
// Schedule update. update() will be invoked on the gui thread.
QMetaObject::invokeMethod(this, "update");
// TODO: remove later, it's only connected by DriverView.
emit vipcThreadFrameReceived(buf); emit vipcThreadFrameReceived(buf);
} }
} }

@ -8,7 +8,6 @@
#include <QThread> #include <QThread>
#include "cereal/visionipc/visionipc_client.h" #include "cereal/visionipc/visionipc_client.h"
#include "selfdrive/camerad/cameras/camera_common.h" #include "selfdrive/camerad/cameras/camera_common.h"
#include "selfdrive/common/visionimg.h"
#include "selfdrive/ui/ui.h" #include "selfdrive/ui/ui.h"
class CameraViewWidget : public QOpenGLWidget, protected QOpenGLFunctions { class CameraViewWidget : public QOpenGLWidget, protected QOpenGLFunctions {
@ -45,10 +44,9 @@ protected:
bool zoomed_view; bool zoomed_view;
std::mutex lock; std::mutex lock;
int latest_texture_id = -1; VisionBuf *latest_frame = nullptr;
GLuint frame_vao, frame_vbo, frame_ibo; GLuint frame_vao, frame_vbo, frame_ibo;
mat4 frame_mat; mat4 frame_mat;
std::unique_ptr<EGLImageTexture> texture[UI_BUF_COUNT];
std::unique_ptr<WaitFence> wait_fence; std::unique_ptr<WaitFence> wait_fence;
std::unique_ptr<QOpenGLShaderProgram> program; std::unique_ptr<QOpenGLShaderProgram> program;
QColor bg = QColor("#000000"); QColor bg = QColor("#000000");
@ -59,6 +57,8 @@ protected:
std::atomic<VisionStreamType> stream_type; std::atomic<VisionStreamType> stream_type;
QThread *vipc_thread = nullptr; QThread *vipc_thread = nullptr;
GLuint textures[3];
protected slots: protected slots:
void vipcConnected(VisionIpcClient *vipc_client); void vipcConnected(VisionIpcClient *vipc_client);
}; };

@ -15,7 +15,6 @@ int main(int argc, char *argv[]) {
{"no-loop", REPLAY_FLAG_NO_LOOP, "stop at the end of the route"}, {"no-loop", REPLAY_FLAG_NO_LOOP, "stop at the end of the route"},
{"no-cache", REPLAY_FLAG_NO_FILE_CACHE, "turn off local cache"}, {"no-cache", REPLAY_FLAG_NO_FILE_CACHE, "turn off local cache"},
{"qcam", REPLAY_FLAG_QCAMERA, "load qcamera"}, {"qcam", REPLAY_FLAG_QCAMERA, "load qcamera"},
{"yuv", REPLAY_FLAG_SEND_YUV, "send yuv frame"},
{"no-hw-decoder", REPLAY_FLAG_NO_HW_DECODER, "disable HW video decoding"}, {"no-hw-decoder", REPLAY_FLAG_NO_HW_DECODER, "disable HW video decoding"},
{"no-vipc", REPLAY_FLAG_NO_VIPC, "do not output video"}, {"no-vipc", REPLAY_FLAG_NO_VIPC, "do not output video"},
}; };

@ -289,7 +289,7 @@ void Replay::startStream(const Segment *cur_segment) {
camera_size[type] = {fr->width, fr->height}; camera_size[type] = {fr->width, fr->height};
} }
} }
camera_server_ = std::make_unique<CameraServer>(camera_size, hasFlag(REPLAY_FLAG_SEND_YUV)); camera_server_ = std::make_unique<CameraServer>(camera_size, true);
} }
// start stream thread // start stream thread

@ -17,7 +17,6 @@ enum REPLAY_FLAGS {
REPLAY_FLAG_NO_LOOP = 0x0010, REPLAY_FLAG_NO_LOOP = 0x0010,
REPLAY_FLAG_NO_FILE_CACHE = 0x0020, REPLAY_FLAG_NO_FILE_CACHE = 0x0020,
REPLAY_FLAG_QCAMERA = 0x0040, REPLAY_FLAG_QCAMERA = 0x0040,
REPLAY_FLAG_SEND_YUV = 0x0080,
REPLAY_FLAG_NO_HW_DECODER = 0x0100, REPLAY_FLAG_NO_HW_DECODER = 0x0100,
REPLAY_FLAG_FULL_SPEED = 0x0200, REPLAY_FLAG_FULL_SPEED = 0x0200,
REPLAY_FLAG_NO_VIPC = 0x0400, REPLAY_FLAG_NO_VIPC = 0x0400,

@ -19,15 +19,16 @@ int main(int argc, char *argv[]) {
{ {
QHBoxLayout *hlayout = new QHBoxLayout(); QHBoxLayout *hlayout = new QHBoxLayout();
layout->addLayout(hlayout); layout->addLayout(hlayout);
hlayout->addWidget(new CameraViewWidget("navd", VISION_STREAM_RGB_MAP, false)); // TODO: make mapd output YUV
hlayout->addWidget(new CameraViewWidget("camerad", VISION_STREAM_RGB_ROAD, false)); // hlayout->addWidget(new CameraViewWidget("navd", VISION_STREAM_MAP, false));
hlayout->addWidget(new CameraViewWidget("camerad", VISION_STREAM_ROAD, false));
} }
{ {
QHBoxLayout *hlayout = new QHBoxLayout(); QHBoxLayout *hlayout = new QHBoxLayout();
layout->addLayout(hlayout); layout->addLayout(hlayout);
hlayout->addWidget(new CameraViewWidget("camerad", VISION_STREAM_RGB_DRIVER, false)); hlayout->addWidget(new CameraViewWidget("camerad", VISION_STREAM_DRIVER, false));
hlayout->addWidget(new CameraViewWidget("camerad", VISION_STREAM_RGB_WIDE_ROAD, false)); hlayout->addWidget(new CameraViewWidget("camerad", VISION_STREAM_WIDE_ROAD, false));
} }
return a.exec(); return a.exec();

Loading…
Cancel
Save