diff --git a/selfdrive/ui/qt/offroad/driverview.cc b/selfdrive/ui/qt/offroad/driverview.cc index db2a4d6b40..f988d193e3 100644 --- a/selfdrive/ui/qt/offroad/driverview.cc +++ b/selfdrive/ui/qt/offroad/driverview.cc @@ -16,7 +16,7 @@ DriverViewWindow::DriverViewWindow(QWidget* parent) : QWidget(parent) { layout->addWidget(cameraView); scene = new DriverViewScene(this); - connect(cameraView, &CameraViewWidget::frameUpdated, scene, &DriverViewScene::frameUpdated); + connect(cameraView, &CameraViewWidget::vipcThreadFrameReceived, scene, &DriverViewScene::frameUpdated); layout->addWidget(scene); layout->setCurrentWidget(scene); } diff --git a/selfdrive/ui/qt/widgets/cameraview.cc b/selfdrive/ui/qt/widgets/cameraview.cc index 3d7c375590..19eb4ab770 100644 --- a/selfdrive/ui/qt/widgets/cameraview.cc +++ b/selfdrive/ui/qt/widgets/cameraview.cc @@ -91,13 +91,11 @@ mat4 get_fit_view_transform(float widget_aspect_ratio, float frame_aspect_ratio) } // namespace -CameraViewWidget::CameraViewWidget(VisionStreamType stream_type, bool zoom, QWidget* parent) : - stream_type(stream_type), zoomed_view(zoom), QOpenGLWidget(parent) { +CameraViewWidget::CameraViewWidget(VisionStreamType type, bool zoom, QWidget* parent) : + stream_type(type), zoomed_view(zoom), QOpenGLWidget(parent) { setAttribute(Qt::WA_OpaquePaintEvent); - - QTimer *t = new QTimer(this); - connect(t, &QTimer::timeout, this, &CameraViewWidget::updateFrame); - t->start(10); + connect(this, &CameraViewWidget::vipcThreadConnected, this, &CameraViewWidget::vipcConnected, Qt::BlockingQueuedConnection); + connect(this, &CameraViewWidget::vipcThreadFrameReceived, this, &CameraViewWidget::vipcFrameReceived); } CameraViewWidget::~CameraViewWidget() { @@ -148,34 +146,20 @@ void CameraViewWidget::initializeGL() { glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(frame_indicies), frame_indicies, GL_STATIC_DRAW); glBindBuffer(GL_ARRAY_BUFFER, 0); glBindVertexArray(0); - - setStreamType(stream_type); } - -void CameraViewWidget::hideEvent(QHideEvent *event) { - vipc_client->connected = false; +void CameraViewWidget::showEvent(QShowEvent *event) { latest_frame = nullptr; + vipc_thread = new QThread(); + connect(vipc_thread, &QThread::started, [=]() { vipcThread(); }); + connect(vipc_thread, &QThread::finished, vipc_thread, &QObject::deleteLater); + vipc_thread->start(); } -void CameraViewWidget::mouseReleaseEvent(QMouseEvent *event) { - emit clicked(); -} - -void CameraViewWidget::resizeGL(int w, int h) { - updateFrameMat(w, h); -} - -void CameraViewWidget::setStreamType(VisionStreamType type) { - if (!vipc_client || type != stream_type) { - stream_type = type; - vipc_client.reset(new VisionIpcClient("camerad", stream_type, true)); - updateFrameMat(width(), height()); - } -} - -void CameraViewWidget::setBackgroundColor(QColor color) { - bg = color; +void CameraViewWidget::hideEvent(QHideEvent *event) { + vipc_thread->requestInterruption(); + vipc_thread->quit(); + vipc_thread->wait(); } void CameraViewWidget::updateFrameMat(int w, int h) { @@ -199,10 +183,10 @@ void CameraViewWidget::updateFrameMat(int w, int h) { }}; frame_mat = matmul(device_transform, frame_transform); } - } else if (vipc_client->connected) { + } else if (stream_width > 0 && stream_height > 0) { // fit frame to widget size float widget_aspect_ratio = (float)width() / height(); - float frame_aspect_ratio = (float)vipc_client->buffers[0].width / vipc_client->buffers[0].height; + float frame_aspect_ratio = (float)stream_width / stream_height; frame_mat = matmul(device_transform, get_fit_view_transform(widget_aspect_ratio, frame_aspect_ratio)); } } @@ -222,8 +206,8 @@ void CameraViewWidget::paintGL() { glBindTexture(GL_TEXTURE_2D, texture[latest_frame->idx]->frame_tex); if (!Hardware::EON()) { // this is handled in ion on QCOM - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, latest_frame->width, latest_frame->height, - 0, GL_RGB, GL_UNSIGNED_BYTE, latest_frame->addr); + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, latest_frame->width, latest_frame->height, + GL_RGB, GL_UNSIGNED_BYTE, latest_frame->addr); } glUseProgram(program->programId()); @@ -237,40 +221,52 @@ void CameraViewWidget::paintGL() { glBindVertexArray(0); } -void CameraViewWidget::updateFrame() { - if (!isVisible()) { - return; +void CameraViewWidget::vipcConnected(VisionIpcClient *vipc_client) { + makeCurrent(); + for (int i = 0; i < vipc_client->num_buffers; i++) { + texture[i].reset(new EGLImageTexture(&vipc_client->buffers[i])); + + glBindTexture(GL_TEXTURE_2D, texture[i]->frame_tex); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + + // BGR + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_R, GL_BLUE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_G, GL_GREEN); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_B, GL_RED); + assert(glGetError() == GL_NO_ERROR); } + latest_frame = nullptr; + stream_width = vipc_client->buffers[0].width; + stream_height = vipc_client->buffers[0].height; + updateFrameMat(width(), height()); +} - if (!vipc_client->connected) { - makeCurrent(); - if (vipc_client->connect(false)) { - // init vision - for (int i = 0; i < vipc_client->num_buffers; i++) { - texture[i].reset(new EGLImageTexture(&vipc_client->buffers[i])); - - glBindTexture(GL_TEXTURE_2D, texture[i]->frame_tex); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); - - // BGR - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_R, GL_BLUE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_G, GL_GREEN); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_B, GL_RED); - assert(glGetError() == GL_NO_ERROR); +void CameraViewWidget::vipcFrameReceived(VisionBuf *buf) { + latest_frame = buf; + update(); +} + +void CameraViewWidget::vipcThread() { + VisionStreamType cur_stream_type = stream_type; + std::unique_ptr vipc_client; + + while (!QThread::currentThread()->isInterruptionRequested()) { + if (!vipc_client || cur_stream_type != stream_type) { + cur_stream_type = stream_type; + vipc_client.reset(new VisionIpcClient("camerad", cur_stream_type, true)); + } + + if (!vipc_client->connected) { + if (!vipc_client->connect(false)) { + QThread::msleep(100); + continue; } - latest_frame = nullptr; - resizeGL(width(), height()); + emit vipcThreadConnected(vipc_client.get()); } - } - VisionBuf *buf = nullptr; - if (vipc_client->connected) { - buf = vipc_client->recv(nullptr, 0); - if (buf != nullptr) { - latest_frame = buf; - update(); - emit frameUpdated(); + if (VisionBuf *buf = vipc_client->recv(nullptr, 1000)) { + emit vipcThreadFrameReceived(buf); } } } diff --git a/selfdrive/ui/qt/widgets/cameraview.h b/selfdrive/ui/qt/widgets/cameraview.h index 3bf34ac58e..caf686cee8 100644 --- a/selfdrive/ui/qt/widgets/cameraview.h +++ b/selfdrive/ui/qt/widgets/cameraview.h @@ -5,9 +5,8 @@ #include #include #include - +#include #include "cereal/visionipc/visionipc_client.h" -#include "selfdrive/common/mat.h" #include "selfdrive/common/visionimg.h" #include "selfdrive/ui/ui.h" @@ -18,33 +17,38 @@ public: using QOpenGLWidget::QOpenGLWidget; explicit CameraViewWidget(VisionStreamType stream_type, bool zoom, QWidget* parent = nullptr); ~CameraViewWidget(); - void setStreamType(VisionStreamType type); - void setBackgroundColor(QColor color); + void setStreamType(VisionStreamType type) { stream_type = type; } + void setBackgroundColor(const QColor &color) { bg = color; } signals: void clicked(); - void frameUpdated(); + void vipcThreadConnected(VisionIpcClient *); + void vipcThreadFrameReceived(VisionBuf *); protected: void paintGL() override; - void resizeGL(int w, int h) override; void initializeGL() override; + void resizeGL(int w, int h) override { updateFrameMat(w, h); } + void showEvent(QShowEvent *event) override; void hideEvent(QHideEvent *event) override; - void mouseReleaseEvent(QMouseEvent *event) override; + void mouseReleaseEvent(QMouseEvent *event) override { emit clicked(); } void updateFrameMat(int w, int h); - std::unique_ptr vipc_client; - -protected slots: - void updateFrame(); + void vipcThread(); -private: bool zoomed_view; VisionBuf *latest_frame = nullptr; GLuint frame_vao, frame_vbo, frame_ibo; mat4 frame_mat; std::unique_ptr texture[UI_BUF_COUNT]; QOpenGLShaderProgram *program; - - VisionStreamType stream_type; QColor bg = QColor("#000000"); + + int stream_width = 0; + int stream_height = 0; + std::atomic stream_type; + QThread *vipc_thread = nullptr; + +protected slots: + void vipcConnected(VisionIpcClient *vipc_client); + void vipcFrameReceived(VisionBuf *buf); };