From 140e6248e26cfa370ccdb13fda8edc4fae0321ca Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Thu, 29 Apr 2021 11:18:59 -0700 Subject: [PATCH] UI: refactor GLWindow (#20764) --- selfdrive/ui/SConscript | 2 +- selfdrive/ui/paint.cc | 14 +- selfdrive/ui/paint.hpp | 1 + selfdrive/ui/qt/home.cc | 183 ++------------------------- selfdrive/ui/qt/home.hpp | 58 ++------- selfdrive/ui/qt/offroad/settings.cc | 5 +- selfdrive/ui/qt/onroad.cc | 57 +++++++++ selfdrive/ui/qt/onroad.hpp | 30 +++++ selfdrive/ui/qt/request_repeater.cc | 11 +- selfdrive/ui/qt/request_repeater.hpp | 9 +- selfdrive/ui/qt/sound.hpp | 24 ++-- selfdrive/ui/qt/window.cc | 25 ++-- selfdrive/ui/qt/window.hpp | 4 + selfdrive/ui/ui.cc | 160 ++++++++++++++++++----- selfdrive/ui/ui.hpp | 85 ++++++++++--- 15 files changed, 358 insertions(+), 310 deletions(-) create mode 100644 selfdrive/ui/qt/onroad.cc create mode 100644 selfdrive/ui/qt/onroad.hpp diff --git a/selfdrive/ui/SConscript b/selfdrive/ui/SConscript index 83b2bd67bd..9462276a71 100644 --- a/selfdrive/ui/SConscript +++ b/selfdrive/ui/SConscript @@ -27,7 +27,7 @@ qt_env.Program("qt/text", ["qt/text.cc"], LIBS=qt_libs) qt_env.Program("qt/spinner", ["qt/spinner.cc"], LIBS=base_libs) # build main UI -qt_src = ["main.cc", "ui.cc", "paint.cc", "sidebar.cc", +qt_src = ["main.cc", "ui.cc", "paint.cc", "sidebar.cc", "qt/onroad.cc", "qt/window.cc", "qt/home.cc", "qt/offroad/settings.cc", "qt/offroad/onboarding.cc", "#phonelibs/nanovg/nanovg.c"] diff --git a/selfdrive/ui/paint.cc b/selfdrive/ui/paint.cc index 7b63690adc..046f01f4c7 100644 --- a/selfdrive/ui/paint.cc +++ b/selfdrive/ui/paint.cc @@ -1,9 +1,19 @@ +#include +#include + #include "ui.hpp" +#ifdef __APPLE__ +#include +#define NANOVG_GL3_IMPLEMENTATION +#define nvgCreate nvgCreateGL3 +#else +#include +#define NANOVG_GLES3_IMPLEMENTATION +#define nvgCreate nvgCreateGLES3 +#endif -#include #include "common/util.h" #include "common/timing.h" -#include #define NANOVG_GLES3_IMPLEMENTATION #include "nanovg_gl.h" diff --git a/selfdrive/ui/paint.hpp b/selfdrive/ui/paint.hpp index bc0927a718..eb107de7c0 100644 --- a/selfdrive/ui/paint.hpp +++ b/selfdrive/ui/paint.hpp @@ -1,4 +1,5 @@ #pragma once + #include "ui.hpp" void ui_draw(UIState *s); diff --git a/selfdrive/ui/qt/home.cc b/selfdrive/ui/qt/home.cc index 27b4e7bcc9..ef47430d96 100644 --- a/selfdrive/ui/qt/home.cc +++ b/selfdrive/ui/qt/home.cc @@ -1,9 +1,3 @@ -#include -#include -#include -#include -#include - #include #include #include @@ -13,59 +7,46 @@ #include "common/params.h" #include "common/timing.h" #include "common/swaglog.h" -#include "common/watchdog.h" -#include "selfdrive/hardware/hw.h" #include "home.hpp" -#include "paint.hpp" -#include "qt_window.hpp" #include "widgets/drive_stats.hpp" #include "widgets/setup.hpp" -#define BACKLIGHT_DT 0.25 -#define BACKLIGHT_TS 2.00 -#define BACKLIGHT_OFFROAD 50 - -// HomeWindow: the container for the offroad (OffroadHome) and onroad (GLWindow) UIs +// HomeWindow: the container for the offroad and onroad UIs HomeWindow::HomeWindow(QWidget* parent) : QWidget(parent) { layout = new QStackedLayout(); layout->setStackingMode(QStackedLayout::StackAll); - // onroad UI - glWindow = new GLWindow(this); - layout->addWidget(glWindow); - - // draw offroad UI on top of onroad UI + onroad = new OnroadWindow(this); + layout->addWidget(onroad); + QObject::connect(this, &HomeWindow::update, onroad, &OnroadWindow::update); + QObject::connect(this, &HomeWindow::displayPowerChanged, onroad, &OnroadWindow::setEnabled); + home = new OffroadHome(); layout->addWidget(home); - - QObject::connect(glWindow, SIGNAL(offroadTransition(bool)), home, SLOT(setVisible(bool))); - QObject::connect(glWindow, SIGNAL(offroadTransition(bool)), this, SIGNAL(offroadTransition(bool))); - QObject::connect(glWindow, SIGNAL(screen_shutoff()), this, SIGNAL(closeSettings())); - QObject::connect(this, SIGNAL(openSettings()), home, SLOT(refresh())); + QObject::connect(this, &HomeWindow::openSettings, home, &OffroadHome::refresh); + QObject::connect(this, &HomeWindow::offroadTransition, home, &OffroadHome::setVisible); setLayout(layout); } void HomeWindow::mousePressEvent(QMouseEvent* e) { - UIState* ui_state = &glWindow->ui_state; - if (GLWindow::ui_state.scene.driver_view) { + // TODO: make a nice driver view widget + if (QUIState::ui_state.scene.driver_view) { Params().putBool("IsDriverViewEnabled", false); - GLWindow::ui_state.scene.driver_view = false; + QUIState::ui_state.scene.driver_view = false; return; } - glWindow->wake(); - // Settings button click - if (!ui_state->sidebar_collapsed && settings_btn.ptInRect(e->x(), e->y())) { + if (!QUIState::ui_state.sidebar_collapsed && settings_btn.ptInRect(e->x(), e->y())) { emit openSettings(); } // Handle sidebar collapsing - if (ui_state->scene.started && (e->x() >= ui_state->viz_rect.x - bdr_s)) { - ui_state->sidebar_collapsed = !ui_state->sidebar_collapsed; + if (QUIState::ui_state.scene.started && (e->x() >= QUIState::ui_state.viz_rect.x - bdr_s)) { + QUIState::ui_state.sidebar_collapsed = !QUIState::ui_state.sidebar_collapsed; } } @@ -189,139 +170,3 @@ void OffroadHome::refresh() { } alert_notification->setStyleSheet(style); } - - -// GLWindow: the onroad UI - -static void handle_display_state(UIState* s, bool user_input) { - static int awake_timeout = 0; - awake_timeout = std::max(awake_timeout - 1, 0); - - constexpr float accel_samples = 5*UI_FREQ; - static float accel_prev = 0., gyro_prev = 0.; - - bool should_wake = s->scene.started || s->scene.ignition || user_input; - if (!should_wake) { - // tap detection while display is off - bool accel_trigger = abs(s->scene.accel_sensor - accel_prev) > 0.2; - bool gyro_trigger = abs(s->scene.gyro_sensor - gyro_prev) > 0.15; - should_wake = accel_trigger && gyro_trigger; - gyro_prev = s->scene.gyro_sensor; - accel_prev = (accel_prev * (accel_samples - 1) + s->scene.accel_sensor) / accel_samples; - } - - if (should_wake) { - awake_timeout = 30 * UI_FREQ; - } else if (awake_timeout > 0) { - should_wake = true; - } - - // handle state transition - if (s->awake != should_wake) { - s->awake = should_wake; - Hardware::set_display_power(s->awake); - LOGD("setting display power %d", s->awake); - } -} - -GLWindow::GLWindow(QWidget* parent) : brightness_filter(BACKLIGHT_OFFROAD, BACKLIGHT_TS, BACKLIGHT_DT), QOpenGLWidget(parent) { - timer = new QTimer(this); - QObject::connect(timer, SIGNAL(timeout()), this, SLOT(timerUpdate())); - - backlight_timer = new QTimer(this); - QObject::connect(backlight_timer, SIGNAL(timeout()), this, SLOT(backlightUpdate())); - - brightness_b = Params(true).get("BRIGHTNESS_B").value_or(10.0); - brightness_m = Params(true).get("BRIGHTNESS_M").value_or(0.1); -} - -GLWindow::~GLWindow() { - makeCurrent(); - doneCurrent(); -} - -void GLWindow::initializeGL() { - initializeOpenGLFunctions(); - std::cout << "OpenGL version: " << glGetString(GL_VERSION) << std::endl; - std::cout << "OpenGL vendor: " << glGetString(GL_VENDOR) << std::endl; - std::cout << "OpenGL renderer: " << glGetString(GL_RENDERER) << std::endl; - std::cout << "OpenGL language version: " << glGetString(GL_SHADING_LANGUAGE_VERSION) << std::endl; - - ui_state.sound = &sound; - ui_state.fb_w = vwp_w; - ui_state.fb_h = vwp_h; - ui_init(&ui_state); - - wake(); - - prev_draw_t = millis_since_boot(); - timer->start(1000 / UI_FREQ); - backlight_timer->start(BACKLIGHT_DT * 1000); -} - -void GLWindow::backlightUpdate() { - // Update brightness - float clipped_brightness = std::min(100.0f, (ui_state.scene.light_sensor * brightness_m) + brightness_b); - if (!ui_state.scene.started) { - clipped_brightness = BACKLIGHT_OFFROAD; - } - - int brightness = brightness_filter.update(clipped_brightness); - if (!ui_state.awake) { - brightness = 0; - emit screen_shutoff(); - } - - if (brightness != last_brightness) { - std::thread{Hardware::set_brightness, brightness}.detach(); - } - last_brightness = brightness; -} - -void GLWindow::timerUpdate() { - // Connecting to visionIPC requires opengl to be current - if (!ui_state.vipc_client->connected){ - makeCurrent(); - } - - if (ui_state.scene.started != onroad) { - onroad = ui_state.scene.started; - emit offroadTransition(!onroad); - - // Change timeout to 0 when onroad, this will call timerUpdate continously. - // This puts visionIPC in charge of update frequency, reducing video latency - timer->start(onroad ? 0 : 1000 / UI_FREQ); - } - - handle_display_state(&ui_state, false); - - // scale volume with speed - sound.volume = util::map_val(ui_state.scene.car_state.getVEgo(), 0.f, 20.f, - Hardware::MIN_VOLUME, Hardware::MAX_VOLUME); - - ui_update(&ui_state); - if(GLWindow::ui_state.awake){ - repaint(); - } - watchdog_kick(); -} - -void GLWindow::resizeGL(int w, int h) { - std::cout << "resize " << w << "x" << h << std::endl; -} - -void GLWindow::paintGL() { - ui_draw(&ui_state); - - double cur_draw_t = millis_since_boot(); - double dt = cur_draw_t - prev_draw_t; - if (dt > 66 && onroad && !ui_state.scene.driver_view) { - // warn on sub 15fps - LOGW("slow frame(%llu) time: %.2f", ui_state.sm->frame, dt); - } - prev_draw_t = cur_draw_t; -} - -void GLWindow::wake() { - handle_display_state(&ui_state, true); -} diff --git a/selfdrive/ui/qt/home.hpp b/selfdrive/ui/qt/home.hpp index d19b9d60cb..cc5f6a8a8f 100644 --- a/selfdrive/ui/qt/home.hpp +++ b/selfdrive/ui/qt/home.hpp @@ -1,61 +1,15 @@ #pragma once #include -#include -#include #include #include -#include #include #include -#include "sound.hpp" +#include "onroad.hpp" #include "ui/ui.hpp" -#include "common/util.h" #include "widgets/offroad_alerts.hpp" -// container window for onroad NVG UI -class GLWindow : public QOpenGLWidget, protected QOpenGLFunctions { - Q_OBJECT - -public: - using QOpenGLWidget::QOpenGLWidget; - explicit GLWindow(QWidget* parent = 0); - void wake(); - ~GLWindow(); - - inline static UIState ui_state = {0}; - -signals: - void offroadTransition(bool offroad); - void screen_shutoff(); - -protected: - void initializeGL() override; - void resizeGL(int w, int h) override; - void paintGL() override; - -private: - QTimer* timer; - QTimer* backlight_timer; - - Sound sound; - - bool onroad = true; - double prev_draw_t = 0; - - // TODO: make a nice abstraction to handle embedded device stuff - float brightness_b = 0; - float brightness_m = 0; - float last_brightness = 0; - FirstOrderFilter brightness_filter; - -public slots: - void timerUpdate(); - void backlightUpdate(); -}; - -// offroad home screen class OffroadHome : public QWidget { Q_OBJECT @@ -81,17 +35,21 @@ class HomeWindow : public QWidget { public: explicit HomeWindow(QWidget* parent = 0); - GLWindow* glWindow; signals: void openSettings(); void closeSettings(); + + // forwarded signals + void displayPowerChanged(bool on); void offroadTransition(bool offroad); + void update(const UIState &s); protected: void mousePressEvent(QMouseEvent* e) override; private: - OffroadHome* home; - QStackedLayout* layout; + OffroadHome *home; + OnroadWindow *onroad; + QStackedLayout *layout; }; diff --git a/selfdrive/ui/qt/offroad/settings.cc b/selfdrive/ui/qt/offroad/settings.cc index 0e36d2770d..ea497a82da 100644 --- a/selfdrive/ui/qt/offroad/settings.cc +++ b/selfdrive/ui/qt/offroad/settings.cc @@ -16,8 +16,7 @@ #include "common/params.h" #include "common/util.h" #include "selfdrive/hardware/hw.h" -#include "home.hpp" - +#include "ui.hpp" TogglesPanel::TogglesPanel(QWidget *parent) : QWidget(parent) { QVBoxLayout *toggles_list = new QVBoxLayout(); @@ -116,7 +115,7 @@ DevicePanel::DevicePanel(QWidget* parent) : QWidget(parent) { "Preview the driver facing camera to help optimize device mounting position for best driver monitoring experience. (vehicle must be off)", [=]() { Params().putBool("IsDriverViewEnabled", true); - GLWindow::ui_state.scene.driver_view = true; + QUIState::ui_state.scene.driver_view = true; }, "", this)); QString resetCalibDesc = "openpilot requires the device to be mounted within 4° left or right and within 5° up or down. openpilot is continuously calibrating, resetting is rarely required."; diff --git a/selfdrive/ui/qt/onroad.cc b/selfdrive/ui/qt/onroad.cc new file mode 100644 index 0000000000..696a30e3fa --- /dev/null +++ b/selfdrive/ui/qt/onroad.cc @@ -0,0 +1,57 @@ +#include + +#include "common/timing.h" +#include "common/swaglog.h" + +#include "onroad.hpp" +#include "paint.hpp" + +OnroadWindow::~OnroadWindow() { + makeCurrent(); + doneCurrent(); +} + +void OnroadWindow::initializeGL() { + initializeOpenGLFunctions(); + std::cout << "OpenGL version: " << glGetString(GL_VERSION) << std::endl; + std::cout << "OpenGL vendor: " << glGetString(GL_VENDOR) << std::endl; + std::cout << "OpenGL renderer: " << glGetString(GL_RENDERER) << std::endl; + std::cout << "OpenGL language version: " << glGetString(GL_SHADING_LANGUAGE_VERSION) << std::endl; + + enabled = true; + ui_nvg_init(&QUIState::ui_state); + prev_draw_t = millis_since_boot(); +} + +void OnroadWindow::setEnabled(bool on) { + enabled = on; +} + +void OnroadWindow::update(const UIState &s) { + // Connecting to visionIPC requires opengl to be current + if (s.vipc_client->connected){ + makeCurrent(); + } + + // TODO: will hide do this? + if(enabled) { + repaint(); + } +} + +void OnroadWindow::resizeGL(int w, int h) { + std::cout << "resize " << w << "x" << h << std::endl; +} + +void OnroadWindow::paintGL() { + ui_draw(&QUIState::ui_state); + + double cur_draw_t = millis_since_boot(); + double dt = cur_draw_t - prev_draw_t; + // TODO: check if onroad + if (dt > 66 && QUIState::ui_state.scene.started && !QUIState::ui_state.scene.driver_view) { + // warn on sub 15fps + LOGW("slow frame time: %.2f", dt); + } + prev_draw_t = cur_draw_t; +} diff --git a/selfdrive/ui/qt/onroad.hpp b/selfdrive/ui/qt/onroad.hpp new file mode 100644 index 0000000000..672144e231 --- /dev/null +++ b/selfdrive/ui/qt/onroad.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include +#include +#include + +#include "ui/ui.hpp" + +// container window for the NVG UI +class OnroadWindow : public QOpenGLWidget, protected QOpenGLFunctions { + Q_OBJECT + +public: + using QOpenGLWidget::QOpenGLWidget; + explicit OnroadWindow(QWidget* parent = 0) : QOpenGLWidget(parent) {}; + ~OnroadWindow(); + +protected: + void paintGL() override; + void initializeGL() override; + void resizeGL(int w, int h) override; + +private: + bool enabled; + double prev_draw_t = 0; + +public slots: + void setEnabled(bool on); + void update(const UIState &s); +}; diff --git a/selfdrive/ui/qt/request_repeater.cc b/selfdrive/ui/qt/request_repeater.cc index 6a5844bf47..a0d4569954 100644 --- a/selfdrive/ui/qt/request_repeater.cc +++ b/selfdrive/ui/qt/request_repeater.cc @@ -1,12 +1,13 @@ #include "request_repeater.hpp" -RequestRepeater::RequestRepeater(QObject *parent, QString requestURL, const QString &cache_key, int period_seconds, bool disableWithScreen) : - HttpRequest(parent, requestURL, cache_key), disableWithScreen(disableWithScreen) { - QTimer* timer = new QTimer(this); +RequestRepeater::RequestRepeater(QObject *parent, QString requestURL, const QString &cacheKey, + int period) : HttpRequest(parent, requestURL, cacheKey) { + timer = new QTimer(this); + timer->setTimerType(Qt::VeryCoarseTimer); QObject::connect(timer, &QTimer::timeout, [=](){ - if (!GLWindow::ui_state.scene.started && reply == NULL && (GLWindow::ui_state.awake || !disableWithScreen)) { + if (!QUIState::ui_state.scene.started && QUIState::ui_state.awake && reply == NULL) { sendRequest(requestURL); } }); - timer->start(period_seconds * 1000); + timer->start(period * 1000); } diff --git a/selfdrive/ui/qt/request_repeater.hpp b/selfdrive/ui/qt/request_repeater.hpp index e544e580d9..c0a0a787ec 100644 --- a/selfdrive/ui/qt/request_repeater.hpp +++ b/selfdrive/ui/qt/request_repeater.hpp @@ -1,9 +1,10 @@ #include "api.hpp" -#include "home.hpp" +#include "ui.hpp" class RequestRepeater : public HttpRequest { - public: - RequestRepeater(QObject *parent, QString requestURL, const QString &cache_key = "", int period_seconds = 0, bool disableWithScreen = true); - bool disableWithScreen; + RequestRepeater(QObject *parent, QString requestURL, const QString &cacheKey = "", int period = 0); + +private: + QTimer *timer; }; diff --git a/selfdrive/ui/qt/sound.hpp b/selfdrive/ui/qt/sound.hpp index ab031dcf8c..1f0ce6d5d6 100644 --- a/selfdrive/ui/qt/sound.hpp +++ b/selfdrive/ui/qt/sound.hpp @@ -6,18 +6,6 @@ typedef cereal::CarControl::HUDControl::AudibleAlert AudibleAlert; -static std::map> sound_map { - // AudibleAlert, (file path, loop count) - {AudibleAlert::CHIME_DISENGAGE, {"../assets/sounds/disengaged.wav", 0}}, - {AudibleAlert::CHIME_ENGAGE, {"../assets/sounds/engaged.wav", 0}}, - {AudibleAlert::CHIME_WARNING1, {"../assets/sounds/warning_1.wav", 0}}, - {AudibleAlert::CHIME_WARNING2, {"../assets/sounds/warning_2.wav", 0}}, - {AudibleAlert::CHIME_WARNING2_REPEAT, {"../assets/sounds/warning_2.wav", -1}}, - {AudibleAlert::CHIME_WARNING_REPEAT, {"../assets/sounds/warning_repeat.wav", -1}}, - {AudibleAlert::CHIME_ERROR, {"../assets/sounds/error.wav", 0}}, - {AudibleAlert::CHIME_PROMPT, {"../assets/sounds/error.wav", 0}} -}; - class Sound { public: Sound(); @@ -26,5 +14,17 @@ public: float volume = 0; private: + std::map> sound_map { + // AudibleAlert, (file path, loop count) + {AudibleAlert::CHIME_DISENGAGE, {"../assets/sounds/disengaged.wav", 0}}, + {AudibleAlert::CHIME_ENGAGE, {"../assets/sounds/engaged.wav", 0}}, + {AudibleAlert::CHIME_WARNING1, {"../assets/sounds/warning_1.wav", 0}}, + {AudibleAlert::CHIME_WARNING2, {"../assets/sounds/warning_2.wav", 0}}, + {AudibleAlert::CHIME_WARNING2_REPEAT, {"../assets/sounds/warning_2.wav", -1}}, + {AudibleAlert::CHIME_WARNING_REPEAT, {"../assets/sounds/warning_repeat.wav", -1}}, + {AudibleAlert::CHIME_ERROR, {"../assets/sounds/error.wav", 0}}, + {AudibleAlert::CHIME_PROMPT, {"../assets/sounds/error.wav", 0}} + }; + std::map sounds; }; diff --git a/selfdrive/ui/qt/window.cc b/selfdrive/ui/qt/window.cc index c88a47dd94..88cf782c12 100644 --- a/selfdrive/ui/qt/window.cc +++ b/selfdrive/ui/qt/window.cc @@ -7,25 +7,30 @@ MainWindow::MainWindow(QWidget *parent) : QWidget(parent) { homeWindow = new HomeWindow(this); main_layout->addWidget(homeWindow); + QObject::connect(homeWindow, &HomeWindow::openSettings, this, &MainWindow::openSettings); + QObject::connect(homeWindow, &HomeWindow::closeSettings, this, &MainWindow::closeSettings); + QObject::connect(&qs, &QUIState::uiUpdate, homeWindow, &HomeWindow::update); + QObject::connect(&qs, &QUIState::offroadTransition, homeWindow, &HomeWindow::offroadTransition); + QObject::connect(&device, &Device::displayPowerChanged, homeWindow, &HomeWindow::displayPowerChanged); settingsWindow = new SettingsWindow(this); main_layout->addWidget(settingsWindow); + QObject::connect(settingsWindow, &SettingsWindow::closeSettings, this, &MainWindow::closeSettings); + QObject::connect(&qs, &QUIState::offroadTransition, settingsWindow, &SettingsWindow::offroadTransition); + QObject::connect(settingsWindow, SIGNAL(reviewTrainingGuide()), this, SLOT(reviewTrainingGuide())); onboardingWindow = new OnboardingWindow(this); main_layout->addWidget(onboardingWindow); - QObject::connect(homeWindow, SIGNAL(openSettings()), this, SLOT(openSettings())); - QObject::connect(homeWindow, SIGNAL(closeSettings()), this, SLOT(closeSettings())); - QObject::connect(homeWindow, SIGNAL(offroadTransition(bool)), this, SLOT(offroadTransition(bool))); - QObject::connect(homeWindow, SIGNAL(offroadTransition(bool)), settingsWindow, SIGNAL(offroadTransition(bool))); - QObject::connect(settingsWindow, SIGNAL(closeSettings()), this, SLOT(closeSettings())); - QObject::connect(settingsWindow, SIGNAL(reviewTrainingGuide()), this, SLOT(reviewTrainingGuide())); - - // start at onboarding main_layout->setCurrentWidget(onboardingWindow); QObject::connect(onboardingWindow, SIGNAL(onboardingDone()), this, SLOT(closeSettings())); onboardingWindow->updateActiveScreen(); + device.setAwake(true, true); + QObject::connect(&qs, &QUIState::uiUpdate, &device, &Device::update); + QObject::connect(&qs, &QUIState::offroadTransition, this, &MainWindow::offroadTransition); + QObject::connect(&device, &Device::displayPowerChanged, this, &MainWindow::closeSettings); + // no outline to prevent the focus rectangle setLayout(main_layout); setStyleSheet(R"( @@ -58,11 +63,11 @@ void MainWindow::reviewTrainingGuide() { bool MainWindow::eventFilter(QObject *obj, QEvent *event){ // wake screen on tap if (event->type() == QEvent::MouseButtonPress) { - homeWindow->glWindow->wake(); + device.setAwake(true, true); } - // filter out touches while in android activity #ifdef QCOM + // filter out touches while in android activity const QList filter_events = {QEvent::MouseButtonPress, QEvent::MouseMove, QEvent::TouchBegin, QEvent::TouchUpdate, QEvent::TouchEnd}; if (HardwareEon::launched_activity && filter_events.contains(event->type())) { HardwareEon::check_activity(); diff --git a/selfdrive/ui/qt/window.hpp b/selfdrive/ui/qt/window.hpp index 37cc70722d..fa1ab2c3e1 100644 --- a/selfdrive/ui/qt/window.hpp +++ b/selfdrive/ui/qt/window.hpp @@ -6,6 +6,7 @@ #include "offroad/settings.hpp" #include "offroad/onboarding.hpp" #include "home.hpp" +#include "../ui.hpp" class MainWindow : public QWidget { Q_OBJECT @@ -17,6 +18,9 @@ public: explicit MainWindow(QWidget *parent = 0); private: + Device device; + QUIState qs; + QStackedLayout *main_layout; HomeWindow *homeWindow; SettingsWindow *settingsWindow; diff --git a/selfdrive/ui/ui.cc b/selfdrive/ui/ui.cc index 9cb04e9f60..a07000db71 100644 --- a/selfdrive/ui/ui.cc +++ b/selfdrive/ui/ui.cc @@ -7,8 +7,16 @@ #include "common/util.h" #include "common/swaglog.h" #include "common/visionimg.h" +#include "common/watchdog.h" +#include "hardware/hw.h" #include "ui.hpp" #include "paint.hpp" +#include "qt_window.hpp" + +#define BACKLIGHT_DT 0.25 +#define BACKLIGHT_TS 2.00 +#define BACKLIGHT_OFFROAD 50 + // Projects a point in car to space to the corresponding point in full frame // image space. @@ -45,34 +53,6 @@ static void ui_init_vision(UIState *s) { assert(glGetError() == GL_NO_ERROR); } - -void ui_init(UIState *s) { - s->sm = new SubMaster({ - "modelV2", "controlsState", "liveCalibration", "radarState", "deviceState", "liveLocationKalman", - "pandaState", "carParams", "driverState", "driverMonitoringState", "sensorEvents", "carState", "ubloxGnss", -#ifdef QCOM2 - "roadCameraState", -#endif - }); - - s->scene.started = false; - s->status = STATUS_OFFROAD; - - - s->last_frame = nullptr; - s->wide_camera = false; - -#ifdef QCOM2 - s->wide_camera = Params().getBool("EnableWideCamera"); -#endif - - ui_nvg_init(s); - - s->vipc_client_rear = new VisionIpcClient("camerad", s->wide_camera ? VISION_STREAM_RGB_WIDE : VISION_STREAM_RGB_BACK, true); - s->vipc_client_front = new VisionIpcClient("camerad", VISION_STREAM_RGB_FRONT, true); - s->vipc_client = s->vipc_client_rear; -} - static int get_path_length_idx(const cereal::ModelDataV2::XYZTData::Reader &line, const float path_height) { const auto line_x = line.getX(); int max_idx = 0; @@ -347,11 +327,121 @@ static void update_status(UIState *s) { started_prev = s->scene.started; } -void ui_update(UIState *s) { - update_params(s); - update_sockets(s); - update_state(s); - update_status(s); - update_alert(s); - update_vision(s); + +QUIState::QUIState(QObject *parent) : QObject(parent) { + ui_state.sound = std::make_unique(); + ui_state.sm = std::make_unique>({ + "modelV2", "controlsState", "liveCalibration", "radarState", "deviceState", "liveLocationKalman", + "pandaState", "carParams", "driverState", "driverMonitoringState", "sensorEvents", "carState", "ubloxGnss", +#ifdef QCOM2 + "roadCameraState", +#endif + }); + + ui_state.fb_w = vwp_w; + ui_state.fb_h = vwp_h; + ui_state.scene.started = false; + ui_state.status = STATUS_OFFROAD; + ui_state.last_frame = nullptr; + ui_state.wide_camera = false; + +#ifdef QCOM2 + ui_state.wide_camera = Params().getBool("EnableWideCamera"); +#endif + + ui_state.vipc_client_rear = new VisionIpcClient("camerad", ui_state.wide_camera ? VISION_STREAM_RGB_WIDE : VISION_STREAM_RGB_BACK, true); + ui_state.vipc_client_front = new VisionIpcClient("camerad", VISION_STREAM_RGB_FRONT, true); + ui_state.vipc_client = ui_state.vipc_client_rear; + + // update timer + timer = new QTimer(this); + QObject::connect(timer, SIGNAL(timeout()), this, SLOT(update())); + timer->start(0); +} + +void QUIState::update() { + update_params(&ui_state); + update_sockets(&ui_state); + update_state(&ui_state); + update_status(&ui_state); + update_alert(&ui_state); + update_vision(&ui_state); + + if (ui_state.scene.started != started_prev) { + started_prev = ui_state.scene.started; + emit offroadTransition(!ui_state.scene.started); + + // Change timeout to 0 when onroad, this will call update continously. + // This puts visionIPC in charge of update frequency, reducing video latency + timer->start(ui_state.scene.started ? 0 : 1000 / UI_FREQ); + } + + // scale volume with speed + QUIState::ui_state.sound->volume = util::map_val(ui_state.scene.car_state.getVEgo(), 0.f, 20.f, + Hardware::MIN_VOLUME, Hardware::MAX_VOLUME); + + watchdog_kick(); + emit uiUpdate(ui_state); +} + +Device::Device(QObject *parent) : brightness_filter(BACKLIGHT_OFFROAD, BACKLIGHT_TS, BACKLIGHT_DT), QObject(parent) { + brightness_b = Params(true).get("BRIGHTNESS_B").value_or(10.0); + brightness_m = Params(true).get("BRIGHTNESS_M").value_or(0.1); +} + +void Device::update(const UIState &s) { + updateBrightness(s); + updateWakefulness(s); + + // TODO: remove from UIState and use signals + QUIState::ui_state.awake = awake; +} + +void Device::setAwake(bool on, bool reset) { + if (on != awake) { + awake = on; + Hardware::set_display_power(awake); + LOGD("setting display power %d", awake); + emit displayPowerChanged(awake); + } + + if (reset) { + awake_timeout = 30 * UI_FREQ; + } +} + +void Device::updateBrightness(const UIState &s) { + float clipped_brightness = std::min(100.0f, (s.scene.light_sensor * brightness_m) + brightness_b); + +#ifdef QCOM2 + if (!s.scene.started) { + clipped_brightness = BACKLIGHT_OFFROAD; + } +#endif + + int brightness = brightness_filter.update(clipped_brightness); + if (!awake) { + brightness = 0; + } + + if (brightness != last_brightness) { + std::thread{Hardware::set_brightness, brightness}.detach(); + } + last_brightness = brightness; +} + +void Device::updateWakefulness(const UIState &s) { + awake_timeout = std::max(awake_timeout - 1, 0); + + bool should_wake = s.scene.started || s.scene.ignition; + if (!should_wake) { + // tap detection while display is off + bool accel_trigger = abs(s.scene.accel_sensor - accel_prev) > 0.2; + bool gyro_trigger = abs(s.scene.gyro_sensor - gyro_prev) > 0.15; + should_wake = accel_trigger && gyro_trigger; + gyro_prev = s.scene.gyro_sensor; + accel_prev = (accel_prev * (accel_samples - 1) + s.scene.accel_sensor) / accel_samples; + } + + setAwake(awake_timeout, should_wake); } diff --git a/selfdrive/ui/ui.hpp b/selfdrive/ui/ui.hpp index 91b1820833..07d80b86ef 100644 --- a/selfdrive/ui/ui.hpp +++ b/selfdrive/ui/ui.hpp @@ -1,15 +1,4 @@ #pragma once -#include "messaging.hpp" - -#ifdef __APPLE__ -#include -#define NANOVG_GL3_IMPLEMENTATION -#define nvgCreate nvgCreateGL3 -#else -#include -#define NANOVG_GLES3_IMPLEMENTATION -#define nvgCreate nvgCreateGLES3 -#endif #include #include @@ -19,16 +8,23 @@ #include "nanovg.h" +#include "camerad/cameras/camera_common.h" #include "common/mat.h" #include "common/visionimg.h" #include "common/modeldata.h" #include "common/params.h" #include "common/glutil.h" +#include "common/util.h" #include "common/transformations/orientation.hpp" -#include "qt/sound.hpp" +#include "messaging.hpp" #include "visionipc.h" #include "visionipc_client.h" +#include "qt/sound.hpp" + +#include +#include + #define COLOR_BLACK nvgRGBA(0, 0, 0, 255) #define COLOR_BLACK_ALPHA(x) nvgRGBA(0, 0, 0, x) #define COLOR_WHITE nvgRGBA(255, 255, 255, 255) @@ -37,8 +33,6 @@ #define COLOR_YELLOW nvgRGBA(218, 202, 37, 255) #define COLOR_RED nvgRGBA(201, 34, 49, 255) -#define UI_BUF_COUNT 4 - typedef struct Rect { int x, y, w, h; int centerX() const { return x + w / 2; } @@ -148,9 +142,9 @@ typedef struct UIState { // images std::map images; - SubMaster *sm; + std::unique_ptr sm; - Sound *sound; + std::unique_ptr sound; UIStatus status; UIScene scene; @@ -161,7 +155,6 @@ typedef struct UIState { GLuint frame_vao[2], frame_vbo[2], frame_ibo[2]; mat4 rear_frame_mat, front_frame_mat; - // device state bool awake; bool sidebar_collapsed; @@ -171,5 +164,59 @@ typedef struct UIState { float zoom; } UIState; -void ui_init(UIState *s); -void ui_update(UIState *s); + +class QUIState : public QObject { + Q_OBJECT + +public: + QUIState(QObject* parent = 0); + + // TODO: get rid of this, only use signal + inline static UIState ui_state = {0}; + +signals: + void uiUpdate(const UIState &s); + void offroadTransition(bool offroad); + +private slots: + void update(); + +private: + QTimer *timer; + bool started_prev = true; +}; + + +// device management class + +class Device : public QObject { + Q_OBJECT + +public: + Device(QObject *parent = 0); + +private: + // auto brightness + const float accel_samples = 5*UI_FREQ; + + bool awake; + int awake_timeout = 0; + float accel_prev = 0; + float gyro_prev = 0; + float brightness_b = 0; + float brightness_m = 0; + float last_brightness = 0; + FirstOrderFilter brightness_filter; + + QTimer *timer; + + void updateBrightness(const UIState &s); + void updateWakefulness(const UIState &s); + +signals: + void displayPowerChanged(bool on); + +public slots: + void setAwake(bool on, bool reset); + void update(const UIState &s); +};