sadmen
/
openpilot_comma
mirror of https://github.com/commaai/openpilot.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

UI: use YUV stream instead of RGB (#24317)

Browse Source 
* UI: use YUV stream instead of RGB

* cleanup

* cleanup
pull/24318/head
Joost Wooning 3 years ago committed by GitHub
parent 01968ca61b
commit 43f43b1c64
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 74 additions and 48 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      selfdrive/ui/qt/offroad/driverview.cc
  2. 4
      selfdrive/ui/qt/onroad.cc
  3. 100
      selfdrive/ui/qt/widgets/cameraview.cc
  4. 3
      selfdrive/ui/qt/widgets/cameraview.h
  5. 1
      selfdrive/ui/replay/main.cc
  6. 2
      selfdrive/ui/replay/replay.cc
  7. 1
      selfdrive/ui/replay/replay.h
  8. 9
      selfdrive/ui/watch3.cc

2
selfdrive/ui/qt/offroad/driverview.cc
Unescape View File

@ -12,7 +12,7 @@ DriverViewWindow::DriverViewWindow(QWidget* parent) : QWidget(parent) {
layout = new QStackedLayout(this);
layout->setStackingMode(QStackedLayout::StackAll);
cameraView = new CameraViewWidget("camerad", VISION_STREAM_RGB_DRIVER, true, this);
cameraView = new CameraViewWidget("camerad", VISION_STREAM_DRIVER, true, this);
layout->addWidget(cameraView);
scene = new DriverViewScene(this);

4
selfdrive/ui/qt/onroad.cc
Unescape View File

@ -20,7 +20,7 @@ OnroadWindow::OnroadWindow(QWidget *parent) : QWidget(parent) {
QStackedLayout *road_view_layout = new QStackedLayout;
road_view_layout->setStackingMode(QStackedLayout::StackAll);
nvg = new NvgWindow(VISION_STREAM_RGB_ROAD, this);
nvg = new NvgWindow(VISION_STREAM_ROAD, this);
road_view_layout->addWidget(nvg);
hud = new OnroadHud(this);
road_view_layout->addWidget(hud);
@ -97,7 +97,7 @@ void OnroadWindow::offroadTransition(bool offroad) {
// update stream type
bool wide_cam = Hardware::TICI() && Params().getBool("EnableWideCamera");
nvg->setStreamType(wide_cam ? VISION_STREAM_RGB_WIDE_ROAD : VISION_STREAM_RGB_ROAD);
nvg->setStreamType(wide_cam ? VISION_STREAM_WIDE_ROAD : VISION_STREAM_ROAD);
}
void OnroadWindow::paintEvent(QPaintEvent *event) {

100
selfdrive/ui/qt/widgets/cameraview.cc
Unescape View File

@ -17,10 +17,10 @@ const char frame_vertex_shader[] =
#else
"#version 300 es\n"
#endif
"in vec4 aPosition;\n"
"in vec4 aTexCoord;\n"
"layout(location = 0) in vec4 aPosition;\n"
"layout(location = 1) in vec2 aTexCoord;\n"
"uniform mat4 uTransform;\n"
"out vec4 vTexCoord;\n"
"out vec2 vTexCoord;\n"
"void main() {\n"
" gl_Position = uTransform * aPosition;\n"
" vTexCoord = aTexCoord;\n"
@ -33,11 +33,19 @@ const char frame_fragment_shader[] =
"#version 300 es\n"
"precision mediump float;\n"
#endif
"uniform sampler2D uTexture;\n"
"in vec4 vTexCoord;\n"
"uniform sampler2D uTextureY;\n"
"uniform sampler2D uTextureU;\n"
"uniform sampler2D uTextureV;\n"
"in vec2 vTexCoord;\n"
"out vec4 colorOut;\n"
"void main() {\n"
" colorOut = texture(uTexture, vTexCoord.xy);\n"
" float y = texture(uTextureY, vTexCoord).r;\n"
" float u = texture(uTextureU, vTexCoord).r - 0.5;\n"
" float v = texture(uTextureV, vTexCoord).r - 0.5;\n"
" float r = y + 1.402 * v;\n"
" float g = y - 0.344 * u - 0.714 * v;\n"
" float b = y + 1.772 * u;\n"
" colorOut = vec4(r, g, b, 1.0);\n"
"}\n";
const mat4 device_transform = {{
@ -102,6 +110,7 @@ CameraViewWidget::~CameraViewWidget() {
glDeleteVertexArrays(1, &frame_vao);
glDeleteBuffers(1, &frame_vbo);
glDeleteBuffers(1, &frame_ibo);
glDeleteBuffers(3, textures);
}
doneCurrent();
}
@ -119,7 +128,7 @@ void CameraViewWidget::initializeGL() {
GLint frame_pos_loc = program->attributeLocation("aPosition");
GLint frame_texcoord_loc = program->attributeLocation("aTexCoord");
auto [x1, x2, y1, y2] = stream_type == VISION_STREAM_RGB_DRIVER ? std::tuple(0.f, 1.f, 1.f, 0.f) : std::tuple(1.f, 0.f, 1.f, 0.f);
auto [x1, x2, y1, y2] = stream_type == VISION_STREAM_DRIVER ? std::tuple(0.f, 1.f, 1.f, 0.f) : std::tuple(1.f, 0.f, 1.f, 0.f);
const uint8_t frame_indicies[] = {0, 1, 2, 0, 2, 3};
const float frame_coords[4][4] = {
{-1.0, -1.0, x2, y1}, // bl
@ -144,6 +153,12 @@ void CameraViewWidget::initializeGL() {
glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(frame_indicies), frame_indicies, GL_STATIC_DRAW);
glBindBuffer(GL_ARRAY_BUFFER, 0);
glBindVertexArray(0);
glGenTextures(3, textures);
glUseProgram(program->programId());
glUniform1i(program->uniformLocation("uTextureY"), 0);
glUniform1i(program->uniformLocation("uTextureU"), 1);
glUniform1i(program->uniformLocation("uTextureV"), 2);
}
void CameraViewWidget::showEvent(QShowEvent *event) {
@ -167,12 +182,12 @@ void CameraViewWidget::hideEvent(QHideEvent *event) {
void CameraViewWidget::updateFrameMat(int w, int h) {
if (zoomed_view) {
if (stream_type == VISION_STREAM_RGB_DRIVER) {
if (stream_type == VISION_STREAM_DRIVER) {
frame_mat = matmul(device_transform, get_driver_view_transform(w, h, stream_width, stream_height));
} else {
auto intrinsic_matrix = stream_type == VISION_STREAM_RGB_WIDE_ROAD ? ecam_intrinsic_matrix : fcam_intrinsic_matrix;
auto intrinsic_matrix = stream_type == VISION_STREAM_WIDE_ROAD ? ecam_intrinsic_matrix : fcam_intrinsic_matrix;
float zoom = ZOOM / intrinsic_matrix.v[0];
if (stream_type == VISION_STREAM_RGB_WIDE_ROAD) {
if (stream_type == VISION_STREAM_WIDE_ROAD) {
zoom *= 0.5;
}
float zx = zoom * 2 * intrinsic_matrix.v[2] / width();
@ -209,38 +224,43 @@ void CameraViewWidget::paintGL() {
}
glBindVertexArray(frame_vao);
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, texture[latest_texture_id]->frame_tex);
glUseProgram(program->programId());
glUniform1i(program->uniformLocation("uTexture"), 0);
glUniformMatrix4fv(program->uniformLocation("uTransform"), 1, GL_TRUE, frame_mat.v);
for (int i = 0; i < 3; ++i) {
glActiveTexture(GL_TEXTURE0 + i);
glBindTexture(GL_TEXTURE_2D, textures[i]);
assert(glGetError() == GL_NO_ERROR);
}
glUniformMatrix4fv(program->uniformLocation("uTransform"), 1, GL_TRUE, frame_mat.v);
assert(glGetError() == GL_NO_ERROR);
glEnableVertexAttribArray(0);
glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_BYTE, (const void *)0);
glDisableVertexAttribArray(0);
glBindVertexArray(0);
glBindTexture(GL_TEXTURE_2D, 0);
glActiveTexture(GL_TEXTURE0);
}
void CameraViewWidget::vipcConnected(VisionIpcClient *vipc_client) {
makeCurrent();
for (int i = 0; i < vipc_client->num_buffers; i++) {
texture[i].reset(new EGLImageTexture(&vipc_client->buffers[i]));
glBindTexture(GL_TEXTURE_2D, texture[i]->frame_tex);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
// BGR
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_R, GL_BLUE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_G, GL_GREEN);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_B, GL_RED);
assert(glGetError() == GL_NO_ERROR);
}
latest_texture_id = -1;
stream_width = vipc_client->buffers[0].width;
stream_height = vipc_client->buffers[0].height;
glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
for (int i = 0; i < 3; ++i) {
glBindTexture(GL_TEXTURE_2D, textures[i]);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
int width = i == 0 ? stream_width : stream_width / 2;
int height = i == 0 ? stream_height : stream_height / 2;
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, nullptr);
assert(glGetError() == GL_NO_ERROR);
}
updateFrameMat(width(), height());
}
@ -290,19 +310,25 @@ void CameraViewWidget::vipcThread() {
{
std::lock_guard lk(lock);
void *texture_buffer = gl_buffer->map(QOpenGLBuffer::WriteOnly);
for (int i = 0; i < 3; i++) {
void *texture_buffer = gl_buffer->map(QOpenGLBuffer::WriteOnly);
if (texture_buffer == nullptr) {
LOGE("gl_buffer->map returned nullptr");
continue;
}
if (texture_buffer == nullptr) {
LOGE("gl_buffer->map returned nullptr");
continue;
}
memcpy(texture_buffer, buf->addr, buf->len);
gl_buffer->unmap();
int width = i == 0 ? stream_width : stream_width / 2;
int height = i == 0 ? stream_height : stream_height / 2;
uint8_t* tex_buf[] = {buf->y, buf->u, buf->v};
memcpy(texture_buffer, tex_buf[i], width*height);
gl_buffer->unmap();
// copy pixels from PBO to texture object
glBindTexture(GL_TEXTURE_2D, textures[i]);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height, GL_LUMINANCE, GL_UNSIGNED_BYTE, 0);
}
// copy pixels from PBO to texture object
glBindTexture(GL_TEXTURE_2D, texture[buf->idx]->frame_tex);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, buf->width, buf->height, GL_RGB, GL_UNSIGNED_BYTE, 0);
glBindTexture(GL_TEXTURE_2D, 0);
assert(glGetError() == GL_NO_ERROR);

3
selfdrive/ui/qt/widgets/cameraview.h
Unescape View File

@ -48,7 +48,6 @@ protected:
int latest_texture_id = -1;
GLuint frame_vao, frame_vbo, frame_ibo;
mat4 frame_mat;
std::unique_ptr<EGLImageTexture> texture[UI_BUF_COUNT];
std::unique_ptr<WaitFence> wait_fence;
std::unique_ptr<QOpenGLShaderProgram> program;
QColor bg = QColor("#000000");
@ -59,6 +58,8 @@ protected:
std::atomic<VisionStreamType> stream_type;
QThread *vipc_thread = nullptr;
GLuint textures[3];
protected slots:
void vipcConnected(VisionIpcClient *vipc_client);
};

1
selfdrive/ui/replay/main.cc
Unescape View File

@ -15,7 +15,6 @@ int main(int argc, char *argv[]) {
{"no-loop", REPLAY_FLAG_NO_LOOP, "stop at the end of the route"},
{"no-cache", REPLAY_FLAG_NO_FILE_CACHE, "turn off local cache"},
{"qcam", REPLAY_FLAG_QCAMERA, "load qcamera"},
{"yuv", REPLAY_FLAG_SEND_YUV, "send yuv frame"},
{"no-hw-decoder", REPLAY_FLAG_NO_HW_DECODER, "disable HW video decoding"},
{"no-vipc", REPLAY_FLAG_NO_VIPC, "do not output video"},
};

2
selfdrive/ui/replay/replay.cc
Unescape View File

@ -289,7 +289,7 @@ void Replay::startStream(const Segment *cur_segment) {
camera_size[type] = {fr->width, fr->height};
}
}
camera_server_ = std::make_unique<CameraServer>(camera_size, hasFlag(REPLAY_FLAG_SEND_YUV));
camera_server_ = std::make_unique<CameraServer>(camera_size, true);
}
// start stream thread

1
selfdrive/ui/replay/replay.h
Unescape View File

@ -17,7 +17,6 @@ enum REPLAY_FLAGS {
REPLAY_FLAG_NO_LOOP = 0x0010,
REPLAY_FLAG_NO_FILE_CACHE = 0x0020,
REPLAY_FLAG_QCAMERA = 0x0040,
REPLAY_FLAG_SEND_YUV = 0x0080,
REPLAY_FLAG_NO_HW_DECODER = 0x0100,
REPLAY_FLAG_FULL_SPEED = 0x0200,
REPLAY_FLAG_NO_VIPC = 0x0400,

9
selfdrive/ui/watch3.cc
Unescape View File

@ -19,15 +19,16 @@ int main(int argc, char *argv[]) {
{
QHBoxLayout *hlayout = new QHBoxLayout();
layout->addLayout(hlayout);
hlayout->addWidget(new CameraViewWidget("navd", VISION_STREAM_RGB_MAP, false));
hlayout->addWidget(new CameraViewWidget("camerad", VISION_STREAM_RGB_ROAD, false));
// TODO: make mapd output YUV
// hlayout->addWidget(new CameraViewWidget("navd", VISION_STREAM_MAP, false));
hlayout->addWidget(new CameraViewWidget("camerad", VISION_STREAM_ROAD, false));
}
{
QHBoxLayout *hlayout = new QHBoxLayout();
layout->addLayout(hlayout);
hlayout->addWidget(new CameraViewWidget("camerad", VISION_STREAM_RGB_DRIVER, false));
hlayout->addWidget(new CameraViewWidget("camerad", VISION_STREAM_RGB_WIDE_ROAD, false));
hlayout->addWidget(new CameraViewWidget("camerad", VISION_STREAM_DRIVER, false));
hlayout->addWidget(new CameraViewWidget("camerad", VISION_STREAM_WIDE_ROAD, false));
}
return a.exec();

Write Preview
Loading…
Cancel
Save