Qt/onroad: use CameraViewWidget (#21821)

* use CameraViewWidget

* no timer, updateFrame after frameSwapped

* use QOpenGLShaderProgram

* merge master

* remove that

* new function setStreamType

* continue

* remove showEvent

* cleanup

* cleanup

* little more

* fix black screen on startup

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>
old-commit-hash: a034926264
commatwo_master
Dean Lee 4 years ago committed by GitHub
parent 4844f8196d
commit f4d8621be1
  1. 2
      release/files_common
  2. 1
      selfdrive/common/SConscript
  3. 63
      selfdrive/common/glutil.cc
  4. 21
      selfdrive/common/glutil.h
  5. 136
      selfdrive/ui/paint.cc
  6. 42
      selfdrive/ui/qt/onroad.cc
  7. 18
      selfdrive/ui/qt/onroad.h
  8. 48
      selfdrive/ui/qt/widgets/cameraview.cc
  9. 10
      selfdrive/ui/qt/widgets/cameraview.h
  10. 75
      selfdrive/ui/ui.cc
  11. 24
      selfdrive/ui/ui.h

@ -190,8 +190,6 @@ selfdrive/common/version.h
selfdrive/common/framebuffer.h
selfdrive/common/framebuffer.cc
selfdrive/common/glutil.cc
selfdrive/common/glutil.h
selfdrive/common/touch.[c,h]
selfdrive/common/swaglog.h
selfdrive/common/swaglog.cc

@ -18,7 +18,6 @@ _common = fxn('common', common_libs, LIBS="json11")
files = [
'clutil.cc',
'glutil.cc',
'visionimg.cc',
]

@ -1,63 +0,0 @@
#include "selfdrive/common/glutil.h"
#include <cassert>
#include <cstdio>
#include <cstdlib>
#include <string>
static GLuint load_shader(GLenum shaderType, const char *src) {
GLint status = 0, len = 0;
GLuint shader = glCreateShader(shaderType);
assert(shader != 0);
glShaderSource(shader, 1, &src, NULL);
glCompileShader(shader);
glGetShaderiv(shader, GL_COMPILE_STATUS, &status);
if (!status) {
glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &len);
if (len) {
std::string msg(len, '\0');
glGetShaderInfoLog(shader, len, NULL, msg.data());
fprintf(stderr, "error compiling shader:\n%s\n", msg.c_str());
}
assert(0);
}
return shader;
}
GLShader::GLShader(const char *vert_src, const char *frag_src) {
GLint status = 0, len = 0;
prog = glCreateProgram();
assert(prog != 0);
vert = load_shader(GL_VERTEX_SHADER, vert_src);
frag = load_shader(GL_FRAGMENT_SHADER, frag_src);
glAttachShader(prog, vert);
glAttachShader(prog, frag);
glLinkProgram(prog);
glGetProgramiv(prog, GL_LINK_STATUS, &status);
if (!status) {
glGetProgramiv(prog, GL_INFO_LOG_LENGTH, &len);
if (len) {
std::string msg(len, '\0');
glGetProgramInfoLog(prog, len, NULL, msg.data());
fprintf(stderr, "error linking program:\n%s\n", msg.c_str());
}
assert(0);
}
}
GLShader::~GLShader() {
glDeleteProgram(prog);
glDeleteShader(frag);
glDeleteShader(vert);
}
GLuint GLShader::getUniformLocation(const char *name) {
auto it = uniform_loc_map.find(name);
if (it == uniform_loc_map.end()) {
it = uniform_loc_map.insert(it, {name, glGetUniformLocation(prog, name)});
}
return it->second;
}

@ -1,21 +0,0 @@
#pragma once
#include <map>
#ifdef __APPLE__
#include <OpenGL/gl3.h>
#else
#include <GLES3/gl3.h>
#endif
class GLShader {
public:
GLShader(const char *vert_src, const char *frag_src);
~GLShader();
GLuint getUniformLocation(const char * name);
GLuint prog = 0;
private:
GLuint vert = 0, frag = 0;
std::map<const char*, GLint> uniform_loc_map;
};

@ -1,6 +1,5 @@
#include "selfdrive/ui/paint.h"
#include <algorithm>
#include <cassert>
#ifdef __APPLE__
@ -17,10 +16,8 @@
#include <nanovg_gl.h>
#include <nanovg_gl_utils.h>
#include "selfdrive/common/timing.h"
#include "selfdrive/common/util.h"
#include "selfdrive/hardware/hw.h"
#include "selfdrive/ui/ui.h"
static void ui_draw_text(const UIState *s, float x, float y, const char *string, float size, NVGcolor color, const char *font_name) {
@ -108,31 +105,6 @@ static void ui_draw_line(UIState *s, const line_vertices_data &vd, NVGcolor *col
nvgFill(s->vg);
}
static void draw_vision_frame(UIState *s) {
glBindVertexArray(s->frame_vao);
mat4 *out_mat = &s->rear_frame_mat;
glActiveTexture(GL_TEXTURE0);
if (s->last_frame) {
glBindTexture(GL_TEXTURE_2D, s->texture[s->last_frame->idx]->frame_tex);
if (!Hardware::EON()) {
// this is handled in ion on QCOM
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, s->last_frame->width, s->last_frame->height,
0, GL_RGB, GL_UNSIGNED_BYTE, s->last_frame->addr);
}
}
glUseProgram(s->gl_shader->prog);
glUniform1i(s->gl_shader->getUniformLocation("uTexture"), 0);
glUniformMatrix4fv(s->gl_shader->getUniformLocation("uTransform"), 1, GL_TRUE, out_mat->v);
assert(glGetError() == GL_NO_ERROR);
glEnableVertexAttribArray(0);
glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_BYTE, (const void *)0);
glDisableVertexAttribArray(0);
glBindVertexArray(0);
}
static void ui_draw_vision_lane_lines(UIState *s) {
const UIScene &scene = s->scene;
NVGpaint track_bg;
@ -249,19 +221,14 @@ static void ui_draw_vision(UIState *s) {
}
void ui_draw(UIState *s, int w, int h) {
const bool draw_vision = s->scene.started && s->vipc_client->connected;
glViewport(0, 0, s->fb_w, s->fb_h);
if (draw_vision) {
draw_vision_frame(s);
// Update intrinsics matrix after possible wide camera toggle change
if (s->fb_w != w || s->fb_h != h) {
ui_resize(s, w, h);
}
glEnable(GL_BLEND);
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
// NVG drawing functions - should be no GL inside NVG frame
nvgBeginFrame(s->vg, s->fb_w, s->fb_h, 1.0f);
if (draw_vision) {
ui_draw_vision(s);
}
ui_draw_vision(s);
nvgEndFrame(s->vg);
glDisable(GL_BLEND);
}
@ -296,49 +263,7 @@ void ui_fill_rect(NVGcontext *vg, const Rect &r, const NVGpaint &paint, float ra
fill_rect(vg, r, nullptr, &paint, radius);
}
static const char frame_vertex_shader[] =
#ifdef NANOVG_GL3_IMPLEMENTATION
"#version 150 core\n"
#else
"#version 300 es\n"
#endif
"in vec4 aPosition;\n"
"in vec4 aTexCoord;\n"
"uniform mat4 uTransform;\n"
"out vec4 vTexCoord;\n"
"void main() {\n"
" gl_Position = uTransform * aPosition;\n"
" vTexCoord = aTexCoord;\n"
"}\n";
static const char frame_fragment_shader[] =
#ifdef NANOVG_GL3_IMPLEMENTATION
"#version 150 core\n"
#else
"#version 300 es\n"
#endif
"precision mediump float;\n"
"uniform sampler2D uTexture;\n"
"in vec4 vTexCoord;\n"
"out vec4 colorOut;\n"
"void main() {\n"
" colorOut = texture(uTexture, vTexCoord.xy);\n"
#ifdef QCOM
" vec3 dz = vec3(0.0627f, 0.0627f, 0.0627f);\n"
" colorOut.rgb = ((vec3(1.0f, 1.0f, 1.0f) - dz) * colorOut.rgb / vec3(1.0f, 1.0f, 1.0f)) + dz;\n"
#endif
"}\n";
static const mat4 device_transform = {{
1.0, 0.0, 0.0, 0.0,
0.0, 1.0, 0.0, 0.0,
0.0, 0.0, 1.0, 0.0,
0.0, 0.0, 0.0, 1.0,
}};
void ui_nvg_init(UIState *s) {
// init drawing
// on EON, we enable MSAA
s->vg = Hardware::EON() ? nvgCreate(0) : nvgCreate(NVG_ANTIALIAS | NVG_STENCIL_STROKES | NVG_DEBUG);
assert(s->vg);
@ -363,45 +288,6 @@ void ui_nvg_init(UIState *s) {
s->images[name] = nvgCreateImage(s->vg, file, 1);
assert(s->images[name] != 0);
}
// init gl
s->gl_shader = std::make_unique<GLShader>(frame_vertex_shader, frame_fragment_shader);
GLint frame_pos_loc = glGetAttribLocation(s->gl_shader->prog, "aPosition");
GLint frame_texcoord_loc = glGetAttribLocation(s->gl_shader->prog, "aTexCoord");
glViewport(0, 0, s->fb_w, s->fb_h);
glDisable(GL_DEPTH_TEST);
assert(glGetError() == GL_NO_ERROR);
float x1 = 1.0, x2 = 0.0, y1 = 1.0, y2 = 0.0;
const uint8_t frame_indicies[] = {0, 1, 2, 0, 2, 3};
const float frame_coords[4][4] = {
{-1.0, -1.0, x2, y1}, //bl
{-1.0, 1.0, x2, y2}, //tl
{ 1.0, 1.0, x1, y2}, //tr
{ 1.0, -1.0, x1, y1}, //br
};
glGenVertexArrays(1, &s->frame_vao);
glBindVertexArray(s->frame_vao);
glGenBuffers(1, &s->frame_vbo);
glBindBuffer(GL_ARRAY_BUFFER, s->frame_vbo);
glBufferData(GL_ARRAY_BUFFER, sizeof(frame_coords), frame_coords, GL_STATIC_DRAW);
glEnableVertexAttribArray(frame_pos_loc);
glVertexAttribPointer(frame_pos_loc, 2, GL_FLOAT, GL_FALSE,
sizeof(frame_coords[0]), (const void *)0);
glEnableVertexAttribArray(frame_texcoord_loc);
glVertexAttribPointer(frame_texcoord_loc, 2, GL_FLOAT, GL_FALSE,
sizeof(frame_coords[0]), (const void *)(sizeof(float) * 2));
glGenBuffers(1, &s->frame_ibo);
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, s->frame_ibo);
glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(frame_indicies), frame_indicies, GL_STATIC_DRAW);
glBindBuffer(GL_ARRAY_BUFFER, 0);
glBindVertexArray(0);
ui_resize(s, s->fb_w, s->fb_h);
}
void ui_resize(UIState *s, int width, int height) {
@ -409,25 +295,11 @@ void ui_resize(UIState *s, int width, int height) {
s->fb_h = height;
auto intrinsic_matrix = s->wide_camera ? ecam_intrinsic_matrix : fcam_intrinsic_matrix;
float zoom = ZOOM / intrinsic_matrix.v[0];
if (s->wide_camera) {
zoom *= 0.5;
}
float zx = zoom * 2 * intrinsic_matrix.v[2] / width;
float zy = zoom * 2 * intrinsic_matrix.v[5] / height;
const mat4 frame_transform = {{
zx, 0.0, 0.0, 0.0,
0.0, zy, 0.0, -y_offset / height * 2,
0.0, 0.0, 1.0, 0.0,
0.0, 0.0, 0.0, 1.0,
}};
s->rear_frame_mat = matmul(device_transform, frame_transform);
// Apply transformation such that video pixel coordinates match video
// 1) Put (0, 0) in the middle of the video
nvgTranslate(s->vg, width / 2, height / 2 + y_offset);

@ -1,9 +1,7 @@
#include "selfdrive/ui/qt/onroad.h"
#include <iostream>
#include <QDebug>
#include "selfdrive/common/swaglog.h"
#include "selfdrive/common/timing.h"
#include "selfdrive/ui/paint.h"
#include "selfdrive/ui/qt/util.h"
@ -11,6 +9,7 @@
#include "selfdrive/ui/qt/maps/map.h"
#endif
OnroadWindow::OnroadWindow(QWidget *parent) : QWidget(parent) {
QVBoxLayout *main_layout = new QVBoxLayout(this);
main_layout->setMargin(bdr_s);
@ -18,8 +17,7 @@ OnroadWindow::OnroadWindow(QWidget *parent) : QWidget(parent) {
stacked_layout->setStackingMode(QStackedLayout::StackAll);
main_layout->addLayout(stacked_layout);
// old UI on bottom
nvg = new NvgWindow(this);
nvg = new NvgWindow(VISION_STREAM_RGB_BACK, this);
QObject::connect(this, &OnroadWindow::updateStateSignal, nvg, &NvgWindow::updateState);
QWidget * split_wrapper = new QWidget;
@ -100,6 +98,10 @@ void OnroadWindow::offroadTransition(bool offroad) {
#endif
alerts->updateAlert({}, bg);
// update stream type
bool wide_cam = Hardware::TICI() && Params().getBool("EnableWideCamera");
nvg->setStreamType(wide_cam ? VISION_STREAM_RGB_WIDE : VISION_STREAM_RGB_BACK);
}
void OnroadWindow::paintEvent(QPaintEvent *event) {
@ -168,18 +170,8 @@ void OnroadAlerts::paintEvent(QPaintEvent *event) {
}
}
NvgWindow::NvgWindow(QWidget *parent) : QOpenGLWidget(parent) {
setAttribute(Qt::WA_OpaquePaintEvent);
}
NvgWindow::~NvgWindow() {
makeCurrent();
doneCurrent();
}
void NvgWindow::initializeGL() {
initializeOpenGLFunctions();
CameraViewWidget::initializeGL();
qInfo() << "OpenGL version:" << QString((const char*)glGetString(GL_VERSION));
qInfo() << "OpenGL vendor:" << QString((const char*)glGetString(GL_VENDOR));
qInfo() << "OpenGL renderer:" << QString((const char*)glGetString(GL_RENDERER));
@ -190,21 +182,19 @@ void NvgWindow::initializeGL() {
}
void NvgWindow::updateState(const UIState &s) {
// Connecting to visionIPC requires opengl to be current
if (s.vipc_client->connected) {
makeCurrent();
}
if (isVisible() != s.vipc_client->connected) {
setVisible(s.vipc_client->connected);
// TODO: make camerad startup faster then remove this
if (s.scene.started) {
if (isVisible() != vipc_client->connected) {
setVisible(vipc_client->connected);
}
if (!isVisible()) {
updateFrame();
}
}
repaint();
}
void NvgWindow::resizeGL(int w, int h) {
ui_resize(&QUIState::ui_state, w, h);
}
void NvgWindow::paintGL() {
CameraViewWidget::paintGL();
ui_draw(&QUIState::ui_state, width(), height());
double cur_draw_t = millis_since_boot();

@ -1,12 +1,9 @@
#pragma once
#include <QOpenGLFunctions>
#include <QOpenGLWidget>
#include <QStackedLayout>
#include <QWidget>
#include "cereal/gen/cpp/log.capnp.h"
#include "selfdrive/ui/qt/qt_window.h"
#include "selfdrive/ui/qt/widgets/cameraview.h"
#include "selfdrive/ui/ui.h"
@ -28,24 +25,17 @@ private:
};
// container window for the NVG UI
class NvgWindow : public QOpenGLWidget, protected QOpenGLFunctions {
class NvgWindow : public CameraViewWidget {
Q_OBJECT
public:
using QOpenGLWidget::QOpenGLWidget;
explicit NvgWindow(QWidget* parent = 0);
~NvgWindow();
explicit NvgWindow(VisionStreamType type, QWidget* parent = 0) : CameraViewWidget(type, true, parent) {}
void updateState(const UIState &s);
protected:
void paintGL() override;
void initializeGL() override;
void resizeGL(int w, int h) override;
private:
double prev_draw_t = 0;
public slots:
void updateState(const UIState &s);
};
// container for all onroad widgets

@ -1,6 +1,7 @@
#include "selfdrive/ui/qt/qt_window.h"
#include "selfdrive/ui/qt/widgets/cameraview.h"
#include "selfdrive/common/swaglog.h"
namespace {
const char frame_vertex_shader[] =
@ -95,9 +96,7 @@ mat4 get_fit_view_transform(float widget_aspect_ratio, float frame_aspect_ratio)
CameraViewWidget::CameraViewWidget(VisionStreamType stream_type, bool zoom, QWidget* parent) :
stream_type(stream_type), zoomed_view(zoom), QOpenGLWidget(parent) {
setAttribute(Qt::WA_OpaquePaintEvent);
timer = new QTimer(this);
connect(timer, &QTimer::timeout, this, &CameraViewWidget::updateFrame);
connect(this, &QOpenGLWidget::frameSwapped, this, &CameraViewWidget::updateFrame);
}
CameraViewWidget::~CameraViewWidget() {
@ -113,9 +112,15 @@ CameraViewWidget::~CameraViewWidget() {
void CameraViewWidget::initializeGL() {
initializeOpenGLFunctions();
gl_shader = std::make_unique<GLShader>(frame_vertex_shader, frame_fragment_shader);
GLint frame_pos_loc = glGetAttribLocation(gl_shader->prog, "aPosition");
GLint frame_texcoord_loc = glGetAttribLocation(gl_shader->prog, "aTexCoord");
program = new QOpenGLShaderProgram(context());
bool ret = program->addShaderFromSourceCode(QOpenGLShader::Vertex, frame_vertex_shader);
assert(ret);
ret = program->addShaderFromSourceCode(QOpenGLShader::Fragment, frame_fragment_shader);
assert(ret);
program->link();
GLint frame_pos_loc = program->attributeLocation("aPosition");
GLint frame_texcoord_loc = program->attributeLocation("aTexCoord");
auto [x1, x2, y1, y2] = stream_type == VISION_STREAM_RGB_FRONT ? std::tuple(0.f, 1.f, 1.f, 0.f) : std::tuple(1.f, 0.f, 1.f, 0.f);
const uint8_t frame_indicies[] = {0, 1, 2, 0, 2, 3};
@ -143,15 +148,11 @@ void CameraViewWidget::initializeGL() {
glBindBuffer(GL_ARRAY_BUFFER, 0);
glBindVertexArray(0);
vipc_client = std::make_unique<VisionIpcClient>("camerad", stream_type, true);
setStreamType(stream_type);
}
void CameraViewWidget::showEvent(QShowEvent *event) {
timer->start(0);
}
void CameraViewWidget::hideEvent(QHideEvent *event) {
timer->stop();
vipc_client->connected = false;
latest_frame = nullptr;
}
@ -161,10 +162,18 @@ void CameraViewWidget::mouseReleaseEvent(QMouseEvent *event) {
}
void CameraViewWidget::resizeGL(int w, int h) {
if (!vipc_client->connected) {
return;
updateFrameMat(w, h);
}
void CameraViewWidget::setStreamType(VisionStreamType type) {
if (!vipc_client || type != stream_type) {
stream_type = type;
vipc_client.reset(new VisionIpcClient("camerad", stream_type, true));
updateFrameMat(width(), height());
}
}
void CameraViewWidget::updateFrameMat(int w, int h) {
if (zoomed_view) {
if (stream_type == VISION_STREAM_RGB_FRONT) {
frame_mat = matmul(device_transform, get_driver_view_transform());
@ -212,9 +221,9 @@ void CameraViewWidget::paintGL() {
0, GL_RGB, GL_UNSIGNED_BYTE, latest_frame->addr);
}
glUseProgram(gl_shader->prog);
glUniform1i(gl_shader->getUniformLocation("uTexture"), 0);
glUniformMatrix4fv(gl_shader->getUniformLocation("uTransform"), 1, GL_TRUE, frame_mat.v);
glUseProgram(program->programId());
glUniform1i(program->uniformLocation("uTexture"), 0);
glUniformMatrix4fv(program->uniformLocation("uTransform"), 1, GL_TRUE, frame_mat.v);
assert(glGetError() == GL_NO_ERROR);
glEnableVertexAttribArray(0);
@ -249,8 +258,11 @@ void CameraViewWidget::updateFrame() {
latest_frame = buf;
update();
emit frameUpdated();
} else {
} else if (!Hardware::PC()) {
LOGE("visionIPC receive timeout");
}
} else {
// try to connect again quickly
QTimer::singleShot(1000. / UI_FREQ, this, &CameraViewWidget::updateFrame);
}
}

@ -3,10 +3,10 @@
#include <memory>
#include <QOpenGLFunctions>
#include <QOpenGLShaderProgram>
#include <QOpenGLWidget>
#include "cereal/visionipc/visionipc_client.h"
#include "selfdrive/common/glutil.h"
#include "selfdrive/common/mat.h"
#include "selfdrive/common/visionimg.h"
#include "selfdrive/ui/ui.h"
@ -18,6 +18,7 @@ public:
using QOpenGLWidget::QOpenGLWidget;
explicit CameraViewWidget(VisionStreamType stream_type, bool zoom, QWidget* parent = nullptr);
~CameraViewWidget();
void setStreamType(VisionStreamType type);
signals:
void clicked();
@ -27,9 +28,10 @@ protected:
void paintGL() override;
void resizeGL(int w, int h) override;
void initializeGL() override;
void showEvent(QShowEvent *event) override;
void hideEvent(QHideEvent *event) override;
void mouseReleaseEvent(QMouseEvent *event) override;
void updateFrameMat(int w, int h);
std::unique_ptr<VisionIpcClient> vipc_client;
protected slots:
void updateFrame();
@ -39,10 +41,8 @@ private:
VisionBuf *latest_frame = nullptr;
GLuint frame_vao, frame_vbo, frame_ibo;
mat4 frame_mat;
std::unique_ptr<VisionIpcClient> vipc_client;
std::unique_ptr<EGLImageTexture> texture[UI_BUF_COUNT];
std::unique_ptr<GLShader> gl_shader;
QOpenGLShaderProgram *program;
QTimer* timer;
VisionStreamType stream_type;
};

@ -6,13 +6,9 @@
#include <cmath>
#include <cstdio>
#include "selfdrive/common/swaglog.h"
#include "selfdrive/common/util.h"
#include "selfdrive/common/visionimg.h"
#include "selfdrive/common/watchdog.h"
#include "selfdrive/hardware/hw.h"
#include "selfdrive/ui/paint.h"
#include "selfdrive/ui/qt/qt_window.h"
#define BACKLIGHT_DT 0.05
#define BACKLIGHT_TS 10.00
@ -35,25 +31,6 @@ static bool calib_frame_to_full_frame(const UIState *s, float in_x, float in_y,
return out->x >= -margin && out->x <= s->fb_w + margin && out->y >= -margin && out->y <= s->fb_h + margin;
}
static void ui_init_vision(UIState *s) {
// Invisible until we receive a calibration message.
s->scene.world_objects_visible = false;
for (int i = 0; i < s->vipc_client->num_buffers; i++) {
s->texture[i].reset(new EGLImageTexture(&s->vipc_client->buffers[i]));
glBindTexture(GL_TEXTURE_2D, s->texture[i]->frame_tex);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
// BGR
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_R, GL_BLUE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_G, GL_GREEN);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_B, GL_RED);
}
assert(glGetError() == GL_NO_ERROR);
}
static int get_path_length_idx(const cereal::ModelDataV2::XYZTData::Reader &line, const float path_height) {
const auto line_x = line.getX();
int max_idx = 0;
@ -207,25 +184,6 @@ static void update_params(UIState *s) {
}
}
static void update_vision(UIState *s) {
if (!s->vipc_client->connected && s->scene.started) {
if (s->vipc_client->connect(false)) {
ui_init_vision(s);
}
}
if (s->vipc_client->connected) {
VisionBuf * buf = s->vipc_client->recv();
if (buf != nullptr) {
s->last_frame = buf;
} else if (!Hardware::PC()) {
LOGE("visionIPC receive timeout");
}
} else if (s->scene.started) {
util::sleep_for(1000. / UI_FREQ);
}
}
static void update_status(UIState *s) {
if (s->scene.started && s->sm->updated("controlsState")) {
auto controls_state = (*s->sm)["controlsState"].getControlsState();
@ -245,24 +203,11 @@ static void update_status(UIState *s) {
if (s->scene.started) {
s->status = STATUS_DISENGAGED;
s->scene.started_frame = s->sm->frame;
s->scene.end_to_end = Params().getBool("EndToEndToggle");
s->wide_camera = Hardware::TICI() ? Params().getBool("EnableWideCamera") : false;
// Update intrinsics matrix after possible wide camera toggle change
if (s->vg) {
ui_resize(s, s->fb_w, s->fb_h);
}
// Choose vision ipc client
if (s->wide_camera) {
s->vipc_client = s->vipc_client_wide;
} else {
s->vipc_client = s->vipc_client_rear;
}
} else {
s->vipc_client->connected = false;
}
// Invisible until we receive a calibration message.
s->scene.world_objects_visible = false;
}
started_prev = s->scene.started;
}
@ -274,21 +219,12 @@ QUIState::QUIState(QObject *parent) : QObject(parent) {
"pandaState", "carParams", "driverMonitoringState", "sensorEvents", "carState", "liveLocationKalman",
});
ui_state.fb_w = vwp_w;
ui_state.fb_h = vwp_h;
ui_state.scene.started = false;
ui_state.last_frame = nullptr;
ui_state.wide_camera = Hardware::TICI() ? Params().getBool("EnableWideCamera") : false;
ui_state.vipc_client_rear = new VisionIpcClient("camerad", VISION_STREAM_RGB_BACK, true);
ui_state.vipc_client_wide = new VisionIpcClient("camerad", VISION_STREAM_RGB_WIDE, true);
ui_state.vipc_client = ui_state.vipc_client_rear;
// update timer
timer = new QTimer(this);
QObject::connect(timer, &QTimer::timeout, this, &QUIState::update);
timer->start(0);
timer->start(1000 / UI_FREQ);
}
void QUIState::update() {
@ -296,15 +232,10 @@ void QUIState::update() {
update_sockets(&ui_state);
update_state(&ui_state);
update_status(&ui_state);
update_vision(&ui_state);
if (ui_state.scene.started != started_prev || ui_state.sm->frame == 1) {
started_prev = ui_state.scene.started;
emit offroadTransition(!ui_state.scene.started);
// Change timeout to 0 when onroad, this will call update continuously.
// This puts visionIPC in charge of update frequency, reducing video latency
timer->start(ui_state.scene.started ? 0 : 1000 / UI_FREQ);
}
watchdog_kick();

@ -1,6 +1,5 @@
#pragma once
#include <atomic>
#include <map>
#include <memory>
#include <string>
@ -12,16 +11,12 @@
#include "nanovg.h"
#include "cereal/messaging/messaging.h"
#include "cereal/visionipc/visionipc.h"
#include "cereal/visionipc/visionipc_client.h"
#include "common/transformations/orientation.hpp"
#include "selfdrive/camerad/cameras/camera_common.h"
#include "selfdrive/common/glutil.h"
#include "selfdrive/common/mat.h"
#include "selfdrive/common/modeldata.h"
#include "selfdrive/common/params.h"
#include "selfdrive/common/util.h"
#include "selfdrive/common/visionimg.h"
#define COLOR_BLACK nvgRGBA(0, 0, 0, 255)
#define COLOR_BLACK_ALPHA(x) nvgRGBA(0, 0, 0, x)
@ -123,15 +118,7 @@ typedef struct UIScene {
} UIScene;
typedef struct UIState {
VisionIpcClient * vipc_client;
VisionIpcClient * vipc_client_rear;
VisionIpcClient * vipc_client_wide;
VisionBuf * last_frame;
// framebuffer
int fb_w, fb_h;
// NVG
int fb_w = 0, fb_h = 0;
NVGcontext *vg;
// images
@ -140,14 +127,7 @@ typedef struct UIState {
std::unique_ptr<SubMaster> sm;
UIStatus status;
UIScene scene;
// graphics
std::unique_ptr<GLShader> gl_shader;
std::unique_ptr<EGLImageTexture> texture[UI_BUF_COUNT];
GLuint frame_vao, frame_vbo, frame_ibo;
mat4 rear_frame_mat;
UIScene scene = {};
bool awake;

Loading…
Cancel
Save