UI: refactor GLWindow (#20764)

selfdrive/ui/SConscript

@@ -27,7 +27,7 @@ qt_env.Program("qt/text", ["qt/text.cc"], LIBS=qt_libs)
 qt_env.Program("qt/spinner", ["qt/spinner.cc"], LIBS=base_libs)
 
 # build main UI
-qt_src = ["main.cc", "ui.cc", "paint.cc", "sidebar.cc",
+qt_src = ["main.cc", "ui.cc", "paint.cc", "sidebar.cc", "qt/onroad.cc",
           "qt/window.cc", "qt/home.cc", "qt/offroad/settings.cc",
           "qt/offroad/onboarding.cc", "#phonelibs/nanovg/nanovg.c"]
 

selfdrive/ui/paint.cc

@@ -1,9 +1,19 @@
+#include <assert.h>
+#include <algorithm>
+
 #include "ui.hpp"
+#ifdef __APPLE__
+#include <OpenGL/gl3.h>
+#define NANOVG_GL3_IMPLEMENTATION
+#define nvgCreate nvgCreateGL3
+#else
+#include <GLES3/gl3.h>
+#define NANOVG_GLES3_IMPLEMENTATION
+#define nvgCreate nvgCreateGLES3
+#endif
 
-#include <assert.h>
 #include "common/util.h"
 #include "common/timing.h"
-#include <algorithm>
 
 #define NANOVG_GLES3_IMPLEMENTATION
 #include "nanovg_gl.h"

selfdrive/ui/paint.hpp

@@ -1,4 +1,5 @@
 #pragma once
+
 #include "ui.hpp"
 
 void ui_draw(UIState *s);

selfdrive/ui/qt/home.cc

@@ -1,9 +1,3 @@
-#include <cmath>
-#include <fstream>
-#include <iostream>
-#include <thread>
-#include <exception>
-
 #include <QDateTime>
 #include <QHBoxLayout>
 #include <QMouseEvent>
@@ -13,59 +7,46 @@
 #include "common/params.h"
 #include "common/timing.h"
 #include "common/swaglog.h"
-#include "common/watchdog.h"
-#include "selfdrive/hardware/hw.h"
 
 #include "home.hpp"
-#include "paint.hpp"
-#include "qt_window.hpp"
 #include "widgets/drive_stats.hpp"
 #include "widgets/setup.hpp"
 
-#define BACKLIGHT_DT 0.25
+// HomeWindow: the container for the offroad and onroad UIs
-#define BACKLIGHT_TS 2.00
-#define BACKLIGHT_OFFROAD 50
-
-// HomeWindow: the container for the offroad (OffroadHome) and onroad (GLWindow) UIs
 
 HomeWindow::HomeWindow(QWidget* parent) : QWidget(parent) {
   layout = new QStackedLayout();
   layout->setStackingMode(QStackedLayout::StackAll);
 
-  // onroad UI
+  onroad = new OnroadWindow(this);
-  glWindow = new GLWindow(this);
+  layout->addWidget(onroad);
-  layout->addWidget(glWindow);
+  QObject::connect(this, &HomeWindow::update, onroad, &OnroadWindow::update);
+  QObject::connect(this, &HomeWindow::displayPowerChanged, onroad, &OnroadWindow::setEnabled);
   
-  // draw offroad UI on top of onroad UI
   home = new OffroadHome();
   layout->addWidget(home);
-
+  QObject::connect(this, &HomeWindow::openSettings, home, &OffroadHome::refresh);
-  QObject::connect(glWindow, SIGNAL(offroadTransition(bool)), home, SLOT(setVisible(bool)));
+  QObject::connect(this, &HomeWindow::offroadTransition, home, &OffroadHome::setVisible);
-  QObject::connect(glWindow, SIGNAL(offroadTransition(bool)), this, SIGNAL(offroadTransition(bool)));
-  QObject::connect(glWindow, SIGNAL(screen_shutoff()), this, SIGNAL(closeSettings()));
-  QObject::connect(this, SIGNAL(openSettings()), home, SLOT(refresh()));
 
   setLayout(layout);
 }
 
 void HomeWindow::mousePressEvent(QMouseEvent* e) {
-  UIState* ui_state = &glWindow->ui_state;
+  // TODO: make a nice driver view widget
-  if (GLWindow::ui_state.scene.driver_view) {
+  if (QUIState::ui_state.scene.driver_view) {
     Params().putBool("IsDriverViewEnabled", false);
-    GLWindow::ui_state.scene.driver_view = false;
+    QUIState::ui_state.scene.driver_view = false;
     return;
   }
 
-  glWindow->wake();
-
   // Settings button click
-  if (!ui_state->sidebar_collapsed && settings_btn.ptInRect(e->x(), e->y())) {
+  if (!QUIState::ui_state.sidebar_collapsed && settings_btn.ptInRect(e->x(), e->y())) {
     emit openSettings();
   }
 
   // Handle sidebar collapsing
-  if (ui_state->scene.started && (e->x() >= ui_state->viz_rect.x - bdr_s)) {
+  if (QUIState::ui_state.scene.started && (e->x() >= QUIState::ui_state.viz_rect.x - bdr_s)) {
-    ui_state->sidebar_collapsed = !ui_state->sidebar_collapsed;
+    QUIState::ui_state.sidebar_collapsed = !QUIState::ui_state.sidebar_collapsed;
   }
 }
 
@@ -189,139 +170,3 @@ void OffroadHome::refresh() {
   }
   alert_notification->setStyleSheet(style);
 }
-
-
-// GLWindow: the onroad UI
-
-static void handle_display_state(UIState* s, bool user_input) {
-  static int awake_timeout = 0;
-  awake_timeout = std::max(awake_timeout - 1, 0);
-
-  constexpr float accel_samples = 5*UI_FREQ;
-  static float accel_prev = 0., gyro_prev = 0.;
-
-  bool should_wake = s->scene.started || s->scene.ignition || user_input;
-  if (!should_wake) {
-    // tap detection while display is off
-    bool accel_trigger = abs(s->scene.accel_sensor - accel_prev) > 0.2;
-    bool gyro_trigger = abs(s->scene.gyro_sensor - gyro_prev) > 0.15;
-    should_wake = accel_trigger && gyro_trigger;
-    gyro_prev = s->scene.gyro_sensor;
-    accel_prev = (accel_prev * (accel_samples - 1) + s->scene.accel_sensor) / accel_samples;
-  }
-
-  if (should_wake) {
-    awake_timeout = 30 * UI_FREQ;
-  } else if (awake_timeout > 0) {
-    should_wake = true;
-  }
-
-  // handle state transition
-  if (s->awake != should_wake) {
-    s->awake = should_wake;
-    Hardware::set_display_power(s->awake);
-    LOGD("setting display power %d", s->awake);
-  }
-}
-
-GLWindow::GLWindow(QWidget* parent) : brightness_filter(BACKLIGHT_OFFROAD, BACKLIGHT_TS, BACKLIGHT_DT), QOpenGLWidget(parent) {
-  timer = new QTimer(this);
-  QObject::connect(timer, SIGNAL(timeout()), this, SLOT(timerUpdate()));
-
-  backlight_timer = new QTimer(this);
-  QObject::connect(backlight_timer, SIGNAL(timeout()), this, SLOT(backlightUpdate()));
-
-  brightness_b = Params(true).get<float>("BRIGHTNESS_B").value_or(10.0);
-  brightness_m = Params(true).get<float>("BRIGHTNESS_M").value_or(0.1);
-}
-
-GLWindow::~GLWindow() {
-  makeCurrent();
-  doneCurrent();
-}
-
-void GLWindow::initializeGL() {
-  initializeOpenGLFunctions();
-  std::cout << "OpenGL version: " << glGetString(GL_VERSION) << std::endl;
-  std::cout << "OpenGL vendor: " << glGetString(GL_VENDOR) << std::endl;
-  std::cout << "OpenGL renderer: " << glGetString(GL_RENDERER) << std::endl;
-  std::cout << "OpenGL language version: " << glGetString(GL_SHADING_LANGUAGE_VERSION) << std::endl;
-
-  ui_state.sound = &sound;
-  ui_state.fb_w = vwp_w;
-  ui_state.fb_h = vwp_h;
-  ui_init(&ui_state);
-
-  wake();
-
-  prev_draw_t = millis_since_boot();
-  timer->start(1000 / UI_FREQ);
-  backlight_timer->start(BACKLIGHT_DT * 1000);
-}
-
-void GLWindow::backlightUpdate() {
-  // Update brightness
-  float clipped_brightness = std::min(100.0f, (ui_state.scene.light_sensor * brightness_m) + brightness_b);
-  if (!ui_state.scene.started) {
-    clipped_brightness = BACKLIGHT_OFFROAD;
-  }
-
-  int brightness = brightness_filter.update(clipped_brightness);
-  if (!ui_state.awake) {
-    brightness = 0;
-    emit screen_shutoff();
-  }
-
-  if (brightness != last_brightness) {
-    std::thread{Hardware::set_brightness, brightness}.detach();
-  }
-  last_brightness = brightness;
-}
-
-void GLWindow::timerUpdate() {
-  // Connecting to visionIPC requires opengl to be current
-  if (!ui_state.vipc_client->connected){
-    makeCurrent();
-  }
-
-  if (ui_state.scene.started != onroad) {
-    onroad = ui_state.scene.started;
-    emit offroadTransition(!onroad);
-
-    // Change timeout to 0 when onroad, this will call timerUpdate continously.
-    // This puts visionIPC in charge of update frequency, reducing video latency
-    timer->start(onroad ? 0 : 1000 / UI_FREQ);
-  }
-
-  handle_display_state(&ui_state, false);
-
-  // scale volume with speed
-  sound.volume = util::map_val(ui_state.scene.car_state.getVEgo(), 0.f, 20.f,
-                               Hardware::MIN_VOLUME, Hardware::MAX_VOLUME);
-
-  ui_update(&ui_state);
-  if(GLWindow::ui_state.awake){
-    repaint();
-  }
-  watchdog_kick();
-}
-
-void GLWindow::resizeGL(int w, int h) {
-  std::cout << "resize " << w << "x" << h << std::endl;
-}
-
-void GLWindow::paintGL() {
-  ui_draw(&ui_state);
-
-  double cur_draw_t = millis_since_boot();
-  double dt = cur_draw_t - prev_draw_t;
-  if (dt > 66 && onroad && !ui_state.scene.driver_view) {
-    // warn on sub 15fps
-    LOGW("slow frame(%llu) time: %.2f", ui_state.sm->frame, dt);
-  }
-  prev_draw_t = cur_draw_t;
-}
-
-void GLWindow::wake() {
-  handle_display_state(&ui_state, true);
-}

selfdrive/ui/qt/home.hpp

@@ -1,61 +1,15 @@
 #pragma once
 
 #include <QLabel>
-#include <QOpenGLFunctions>
-#include <QOpenGLWidget>
 #include <QPushButton>
 #include <QStackedLayout>
-#include <QStackedWidget>
 #include <QTimer>
 #include <QWidget>
 
-#include "sound.hpp"
+#include "onroad.hpp"
 #include "ui/ui.hpp"
-#include "common/util.h"
 #include "widgets/offroad_alerts.hpp"
 
-// container window for onroad NVG UI
-class GLWindow : public QOpenGLWidget, protected QOpenGLFunctions {
-  Q_OBJECT
-
-public:
-  using QOpenGLWidget::QOpenGLWidget;
-  explicit GLWindow(QWidget* parent = 0);
-  void wake();
-  ~GLWindow();
-
-  inline static UIState ui_state = {0};
-
-signals:
-  void offroadTransition(bool offroad);
-  void screen_shutoff();
-
-protected:
-  void initializeGL() override;
-  void resizeGL(int w, int h) override;
-  void paintGL() override;
-
-private:
-  QTimer* timer;
-  QTimer* backlight_timer;
-
-  Sound sound;
-
-  bool onroad = true;
-  double prev_draw_t = 0;
-
-  // TODO: make a nice abstraction to handle embedded device stuff
-  float brightness_b = 0;
-  float brightness_m = 0;
-  float last_brightness = 0;
-  FirstOrderFilter brightness_filter;
-
-public slots:
-  void timerUpdate();
-  void backlightUpdate();
-};
-
-// offroad home screen
 class OffroadHome : public QWidget {
   Q_OBJECT
 
@@ -81,17 +35,21 @@ class HomeWindow : public QWidget {
 
 public:
   explicit HomeWindow(QWidget* parent = 0);
-  GLWindow* glWindow;
 
 signals:
   void openSettings();
   void closeSettings();
+
+  // forwarded signals
+  void displayPowerChanged(bool on);
   void offroadTransition(bool offroad);
+  void update(const UIState &s);
 
 protected:
   void mousePressEvent(QMouseEvent* e) override;
 
 private:
   OffroadHome *home;
+  OnroadWindow *onroad;
   QStackedLayout *layout;
 };

selfdrive/ui/qt/offroad/settings.cc

@@ -16,8 +16,7 @@
 #include "common/params.h"
 #include "common/util.h"
 #include "selfdrive/hardware/hw.h"
-#include "home.hpp"
+#include "ui.hpp"
-
 
 TogglesPanel::TogglesPanel(QWidget *parent) : QWidget(parent) {
   QVBoxLayout *toggles_list = new QVBoxLayout();
@@ -116,7 +115,7 @@ DevicePanel::DevicePanel(QWidget* parent) : QWidget(parent) {
                                         "Preview the driver facing camera to help optimize device mounting position for best driver monitoring experience. (vehicle must be off)",
                                         [=]() {
                                            Params().putBool("IsDriverViewEnabled", true);
-                                           GLWindow::ui_state.scene.driver_view = true;
+                                           QUIState::ui_state.scene.driver_view = true;
                                         }, "", this));
 
   QString resetCalibDesc = "openpilot requires the device to be mounted within 4° left or right and within 5° up or down. openpilot is continuously calibrating, resetting is rarely required.";

selfdrive/ui/qt/onroad.cc

@@ -0,0 +1,57 @@
+#include <iostream>
+
+#include "common/timing.h"
+#include "common/swaglog.h"
+
+#include "onroad.hpp"
+#include "paint.hpp"
+
+OnroadWindow::~OnroadWindow() {
+  makeCurrent();
+  doneCurrent();
+}
+
+void OnroadWindow::initializeGL() {
+  initializeOpenGLFunctions();
+  std::cout << "OpenGL version: " << glGetString(GL_VERSION) << std::endl;
+  std::cout << "OpenGL vendor: " << glGetString(GL_VENDOR) << std::endl;
+  std::cout << "OpenGL renderer: " << glGetString(GL_RENDERER) << std::endl;
+  std::cout << "OpenGL language version: " << glGetString(GL_SHADING_LANGUAGE_VERSION) << std::endl;
+
+  enabled = true;
+  ui_nvg_init(&QUIState::ui_state);
+  prev_draw_t = millis_since_boot();
+}
+
+void OnroadWindow::setEnabled(bool on) {
+  enabled = on;
+}
+
+void OnroadWindow::update(const UIState &s) {
+  // Connecting to visionIPC requires opengl to be current
+  if (s.vipc_client->connected){
+    makeCurrent();
+  }
+
+  // TODO: will hide do this?
+  if(enabled) {
+    repaint();
+  }
+}
+
+void OnroadWindow::resizeGL(int w, int h) {
+  std::cout << "resize " << w << "x" << h << std::endl;
+}
+
+void OnroadWindow::paintGL() {
+  ui_draw(&QUIState::ui_state);
+
+  double cur_draw_t = millis_since_boot();
+  double dt = cur_draw_t - prev_draw_t;
+  // TODO: check if onroad
+  if (dt > 66 && QUIState::ui_state.scene.started && !QUIState::ui_state.scene.driver_view) {
+    // warn on sub 15fps
+    LOGW("slow frame time: %.2f", dt);
+  }
+  prev_draw_t = cur_draw_t;
+}

selfdrive/ui/qt/onroad.hpp

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <QOpenGLFunctions>
+#include <QOpenGLWidget>
+#include <QTimer>
+
+#include "ui/ui.hpp"
+
+// container window for the NVG UI
+class OnroadWindow : public QOpenGLWidget, protected QOpenGLFunctions {
+  Q_OBJECT
+
+public:
+  using QOpenGLWidget::QOpenGLWidget;
+  explicit OnroadWindow(QWidget* parent = 0) : QOpenGLWidget(parent) {};
+  ~OnroadWindow();
+
+protected:
+  void paintGL() override;
+  void initializeGL() override;
+  void resizeGL(int w, int h) override;
+
+private:
+  bool enabled;
+  double prev_draw_t = 0;
+
+public slots:
+  void setEnabled(bool on);
+  void update(const UIState &s);
+};

selfdrive/ui/qt/request_repeater.cc

@@ -1,12 +1,13 @@
 #include "request_repeater.hpp"
 
-RequestRepeater::RequestRepeater(QObject *parent, QString requestURL, const QString &cache_key, int period_seconds, bool disableWithScreen) :
+RequestRepeater::RequestRepeater(QObject *parent, QString requestURL, const QString &cacheKey,
-  HttpRequest(parent, requestURL, cache_key), disableWithScreen(disableWithScreen) {
+                                 int period) : HttpRequest(parent, requestURL, cacheKey) {
-  QTimer* timer = new QTimer(this);
+  timer = new QTimer(this);
+  timer->setTimerType(Qt::VeryCoarseTimer);
   QObject::connect(timer, &QTimer::timeout, [=](){
-    if (!GLWindow::ui_state.scene.started && reply == NULL && (GLWindow::ui_state.awake || !disableWithScreen)) {
+    if (!QUIState::ui_state.scene.started && QUIState::ui_state.awake && reply == NULL) {
       sendRequest(requestURL);
     }
   });
-  timer->start(period_seconds * 1000);
+  timer->start(period * 1000);
 }

selfdrive/ui/qt/request_repeater.hpp

@@ -1,9 +1,10 @@
 #include "api.hpp"
-#include "home.hpp"
+#include "ui.hpp"
 
 class RequestRepeater : public HttpRequest {
-
 public:
-  RequestRepeater(QObject *parent, QString requestURL, const QString &cache_key = "", int period_seconds = 0, bool disableWithScreen = true);
+  RequestRepeater(QObject *parent, QString requestURL, const QString &cacheKey = "", int period = 0);
-  bool disableWithScreen;
+
+private:
+  QTimer *timer;
 };

selfdrive/ui/qt/sound.hpp

@@ -6,7 +6,15 @@
 
 typedef cereal::CarControl::HUDControl::AudibleAlert AudibleAlert;
 
-static std::map<AudibleAlert, std::pair<const char *, int>> sound_map {
+class Sound {
+public:
+  Sound();
+  void play(AudibleAlert alert);
+  void stop();
+  float volume = 0;
+
+private:
+  std::map<AudibleAlert, std::pair<QString, int>> sound_map {
     // AudibleAlert, (file path, loop count)
     {AudibleAlert::CHIME_DISENGAGE, {"../assets/sounds/disengaged.wav", 0}},
     {AudibleAlert::CHIME_ENGAGE, {"../assets/sounds/engaged.wav", 0}},
@@ -18,13 +26,5 @@ static std::map<AudibleAlert, std::pair<const char *, int>> sound_map {
     {AudibleAlert::CHIME_PROMPT, {"../assets/sounds/error.wav", 0}}
   };
 
-class Sound {
-public:
-  Sound();
-  void play(AudibleAlert alert);
-  void stop();
-  float volume = 0;
-
-private:
   std::map<AudibleAlert, QSoundEffect> sounds;
 };

selfdrive/ui/qt/window.cc

@@ -7,25 +7,30 @@ MainWindow::MainWindow(QWidget *parent) : QWidget(parent) {
 
   homeWindow = new HomeWindow(this);
   main_layout->addWidget(homeWindow);
+  QObject::connect(homeWindow, &HomeWindow::openSettings, this, &MainWindow::openSettings);
+  QObject::connect(homeWindow, &HomeWindow::closeSettings, this, &MainWindow::closeSettings);
+  QObject::connect(&qs, &QUIState::uiUpdate, homeWindow, &HomeWindow::update);
+  QObject::connect(&qs, &QUIState::offroadTransition, homeWindow, &HomeWindow::offroadTransition);
+  QObject::connect(&device, &Device::displayPowerChanged, homeWindow, &HomeWindow::displayPowerChanged);
 
   settingsWindow = new SettingsWindow(this);
   main_layout->addWidget(settingsWindow);
+  QObject::connect(settingsWindow, &SettingsWindow::closeSettings, this, &MainWindow::closeSettings);
+  QObject::connect(&qs, &QUIState::offroadTransition, settingsWindow, &SettingsWindow::offroadTransition);
+  QObject::connect(settingsWindow, SIGNAL(reviewTrainingGuide()), this, SLOT(reviewTrainingGuide()));
 
   onboardingWindow = new OnboardingWindow(this);
   main_layout->addWidget(onboardingWindow);
 
-  QObject::connect(homeWindow, SIGNAL(openSettings()), this, SLOT(openSettings()));
-  QObject::connect(homeWindow, SIGNAL(closeSettings()), this, SLOT(closeSettings()));
-  QObject::connect(homeWindow, SIGNAL(offroadTransition(bool)), this, SLOT(offroadTransition(bool)));
-  QObject::connect(homeWindow, SIGNAL(offroadTransition(bool)), settingsWindow, SIGNAL(offroadTransition(bool)));
-  QObject::connect(settingsWindow, SIGNAL(closeSettings()), this, SLOT(closeSettings()));
-  QObject::connect(settingsWindow, SIGNAL(reviewTrainingGuide()), this, SLOT(reviewTrainingGuide()));
-
-  // start at onboarding
   main_layout->setCurrentWidget(onboardingWindow);
   QObject::connect(onboardingWindow, SIGNAL(onboardingDone()), this, SLOT(closeSettings()));
   onboardingWindow->updateActiveScreen();
 
+  device.setAwake(true, true);
+  QObject::connect(&qs, &QUIState::uiUpdate, &device, &Device::update);
+  QObject::connect(&qs, &QUIState::offroadTransition, this, &MainWindow::offroadTransition);
+  QObject::connect(&device, &Device::displayPowerChanged, this, &MainWindow::closeSettings);
+
   // no outline to prevent the focus rectangle
   setLayout(main_layout);
   setStyleSheet(R"(
@@ -58,11 +63,11 @@ void MainWindow::reviewTrainingGuide() {
 bool MainWindow::eventFilter(QObject *obj, QEvent *event){
   // wake screen on tap
   if (event->type() == QEvent::MouseButtonPress) {
-    homeWindow->glWindow->wake();
+    device.setAwake(true, true);
   }
 
-  // filter out touches while in android activity
 #ifdef QCOM
+  // filter out touches while in android activity
   const QList<QEvent::Type> filter_events = {QEvent::MouseButtonPress, QEvent::MouseMove, QEvent::TouchBegin, QEvent::TouchUpdate, QEvent::TouchEnd};
   if (HardwareEon::launched_activity && filter_events.contains(event->type())) {
     HardwareEon::check_activity();

selfdrive/ui/qt/window.hpp

@@ -6,6 +6,7 @@
 #include "offroad/settings.hpp"
 #include "offroad/onboarding.hpp"
 #include "home.hpp"
+#include "../ui.hpp"
 
 class MainWindow : public QWidget {
   Q_OBJECT
@@ -17,6 +18,9 @@ public:
   explicit MainWindow(QWidget *parent = 0);
 
 private:
+  Device device;
+  QUIState qs;
+
   QStackedLayout *main_layout;
   HomeWindow *homeWindow;
   SettingsWindow *settingsWindow;

selfdrive/ui/ui.cc

@@ -7,8 +7,16 @@
 #include "common/util.h"
 #include "common/swaglog.h"
 #include "common/visionimg.h"
+#include "common/watchdog.h"
+#include "hardware/hw.h"
 #include "ui.hpp"
 #include "paint.hpp"
+#include "qt_window.hpp"
+
+#define BACKLIGHT_DT 0.25
+#define BACKLIGHT_TS 2.00
+#define BACKLIGHT_OFFROAD 50
+
 
 // Projects a point in car to space to the corresponding point in full frame
 // image space.
@@ -45,34 +53,6 @@ static void ui_init_vision(UIState *s) {
   assert(glGetError() == GL_NO_ERROR);
 }
 
-
-void ui_init(UIState *s) {
-  s->sm = new SubMaster({
-    "modelV2", "controlsState", "liveCalibration", "radarState", "deviceState", "liveLocationKalman",
-    "pandaState", "carParams", "driverState", "driverMonitoringState", "sensorEvents", "carState", "ubloxGnss",
-#ifdef QCOM2
-    "roadCameraState",
-#endif
-  });
-
-  s->scene.started = false;
-  s->status = STATUS_OFFROAD;
-
-
-  s->last_frame = nullptr;
-  s->wide_camera = false;
-
-#ifdef QCOM2
-  s->wide_camera = Params().getBool("EnableWideCamera");
-#endif
-
-  ui_nvg_init(s);
-
-  s->vipc_client_rear = new VisionIpcClient("camerad", s->wide_camera ? VISION_STREAM_RGB_WIDE : VISION_STREAM_RGB_BACK, true);
-  s->vipc_client_front = new VisionIpcClient("camerad", VISION_STREAM_RGB_FRONT, true);
-  s->vipc_client = s->vipc_client_rear;
-}
-
 static int get_path_length_idx(const cereal::ModelDataV2::XYZTData::Reader &line, const float path_height) {
   const auto line_x = line.getX();
   int max_idx = 0;
@@ -347,11 +327,121 @@ static void update_status(UIState *s) {
   started_prev = s->scene.started;
 }
 
-void ui_update(UIState *s) {
+
-  update_params(s);
+QUIState::QUIState(QObject *parent) : QObject(parent) {
-  update_sockets(s);
+  ui_state.sound = std::make_unique<Sound>();
-  update_state(s);
+  ui_state.sm = std::make_unique<SubMaster, const std::initializer_list<const char *>>({
-  update_status(s);
+    "modelV2", "controlsState", "liveCalibration", "radarState", "deviceState", "liveLocationKalman",
-  update_alert(s);
+    "pandaState", "carParams", "driverState", "driverMonitoringState", "sensorEvents", "carState", "ubloxGnss",
-  update_vision(s);
+#ifdef QCOM2
+    "roadCameraState",
+#endif
+  });
+
+  ui_state.fb_w = vwp_w;
+  ui_state.fb_h = vwp_h;
+  ui_state.scene.started = false;
+  ui_state.status = STATUS_OFFROAD;
+  ui_state.last_frame = nullptr;
+  ui_state.wide_camera = false;
+
+#ifdef QCOM2
+  ui_state.wide_camera = Params().getBool("EnableWideCamera");
+#endif
+
+  ui_state.vipc_client_rear = new VisionIpcClient("camerad", ui_state.wide_camera ? VISION_STREAM_RGB_WIDE : VISION_STREAM_RGB_BACK, true);
+  ui_state.vipc_client_front = new VisionIpcClient("camerad", VISION_STREAM_RGB_FRONT, true);
+  ui_state.vipc_client = ui_state.vipc_client_rear;
+
+  // update timer
+  timer = new QTimer(this);
+  QObject::connect(timer, SIGNAL(timeout()), this, SLOT(update()));
+  timer->start(0);
+}
+
+void QUIState::update() {
+  update_params(&ui_state);
+  update_sockets(&ui_state);
+  update_state(&ui_state);
+  update_status(&ui_state);
+  update_alert(&ui_state);
+  update_vision(&ui_state);
+
+  if (ui_state.scene.started != started_prev) {
+    started_prev = ui_state.scene.started;
+    emit offroadTransition(!ui_state.scene.started);
+
+    // Change timeout to 0 when onroad, this will call update continously.
+    // This puts visionIPC in charge of update frequency, reducing video latency
+    timer->start(ui_state.scene.started ? 0 : 1000 / UI_FREQ);
+  }
+
+  // scale volume with speed
+  QUIState::ui_state.sound->volume = util::map_val(ui_state.scene.car_state.getVEgo(), 0.f, 20.f,
+                                                   Hardware::MIN_VOLUME, Hardware::MAX_VOLUME);
+
+  watchdog_kick();
+  emit uiUpdate(ui_state);
+}
+
+Device::Device(QObject *parent) : brightness_filter(BACKLIGHT_OFFROAD, BACKLIGHT_TS, BACKLIGHT_DT), QObject(parent) {
+  brightness_b = Params(true).get<float>("BRIGHTNESS_B").value_or(10.0);
+  brightness_m = Params(true).get<float>("BRIGHTNESS_M").value_or(0.1);
+}
+
+void Device::update(const UIState &s) {
+  updateBrightness(s);
+  updateWakefulness(s);
+
+  // TODO: remove from UIState and use signals
+  QUIState::ui_state.awake = awake;
+}
+
+void Device::setAwake(bool on, bool reset) {
+  if (on != awake) {
+    awake = on;
+    Hardware::set_display_power(awake);
+    LOGD("setting display power %d", awake);
+    emit displayPowerChanged(awake);
+  }
+
+  if (reset) {
+    awake_timeout = 30 * UI_FREQ;
+  }
+}
+
+void Device::updateBrightness(const UIState &s) {
+  float clipped_brightness = std::min(100.0f, (s.scene.light_sensor * brightness_m) + brightness_b);
+
+#ifdef QCOM2
+  if (!s.scene.started) {
+    clipped_brightness = BACKLIGHT_OFFROAD;
+  }
+#endif
+
+  int brightness = brightness_filter.update(clipped_brightness);
+  if (!awake) {
+    brightness = 0;
+  }
+
+  if (brightness != last_brightness) {
+    std::thread{Hardware::set_brightness, brightness}.detach();
+  }
+  last_brightness = brightness;
+}
+
+void Device::updateWakefulness(const UIState &s) {
+  awake_timeout = std::max(awake_timeout - 1, 0);
+
+  bool should_wake = s.scene.started || s.scene.ignition;
+  if (!should_wake) {
+    // tap detection while display is off
+    bool accel_trigger = abs(s.scene.accel_sensor - accel_prev) > 0.2;
+    bool gyro_trigger = abs(s.scene.gyro_sensor - gyro_prev) > 0.15;
+    should_wake = accel_trigger && gyro_trigger;
+    gyro_prev = s.scene.gyro_sensor;
+    accel_prev = (accel_prev * (accel_samples - 1) + s.scene.accel_sensor) / accel_samples;
+  }
+
+  setAwake(awake_timeout, should_wake);
 }

selfdrive/ui/ui.hpp

@@ -1,15 +1,4 @@
 #pragma once
-#include "messaging.hpp"
-
-#ifdef __APPLE__
-#include <OpenGL/gl3.h>
-#define NANOVG_GL3_IMPLEMENTATION
-#define nvgCreate nvgCreateGL3
-#else
-#include <GLES3/gl3.h>
-#define NANOVG_GLES3_IMPLEMENTATION
-#define nvgCreate nvgCreateGLES3
-#endif
 
 #include <atomic>
 #include <map>
@@ -19,16 +8,23 @@
 
 #include "nanovg.h"
 
+#include "camerad/cameras/camera_common.h"
 #include "common/mat.h"
 #include "common/visionimg.h"
 #include "common/modeldata.h"
 #include "common/params.h"
 #include "common/glutil.h"
+#include "common/util.h"
 #include "common/transformations/orientation.hpp"
-#include "qt/sound.hpp"
+#include "messaging.hpp"
 #include "visionipc.h"
 #include "visionipc_client.h"
 
+#include "qt/sound.hpp"
+
+#include <QObject>
+#include <QTimer>
+
 #define COLOR_BLACK nvgRGBA(0, 0, 0, 255)
 #define COLOR_BLACK_ALPHA(x) nvgRGBA(0, 0, 0, x)
 #define COLOR_WHITE nvgRGBA(255, 255, 255, 255)
@@ -37,8 +33,6 @@
 #define COLOR_YELLOW nvgRGBA(218, 202, 37, 255)
 #define COLOR_RED nvgRGBA(201, 34, 49, 255)
 
-#define UI_BUF_COUNT 4
-
 typedef struct Rect {
   int x, y, w, h;
   int centerX() const { return x + w / 2; }
@@ -148,9 +142,9 @@ typedef struct UIState {
   // images
   std::map<std::string, int> images;
 
-  SubMaster *sm;
+  std::unique_ptr<SubMaster> sm;
 
-  Sound *sound;
+  std::unique_ptr<Sound> sound;
   UIStatus status;
   UIScene scene;
 
@@ -161,7 +155,6 @@ typedef struct UIState {
   GLuint frame_vao[2], frame_vbo[2], frame_ibo[2];
   mat4 rear_frame_mat, front_frame_mat;
 
-  // device state
   bool awake;
 
   bool sidebar_collapsed;
@@ -171,5 +164,59 @@ typedef struct UIState {
   float zoom;
 } UIState;
 
-void ui_init(UIState *s);
+
-void ui_update(UIState *s);
+class QUIState : public QObject {
+  Q_OBJECT
+
+public:
+  QUIState(QObject* parent = 0);
+
+  // TODO: get rid of this, only use signal
+  inline static UIState ui_state = {0};
+
+signals:
+  void uiUpdate(const UIState &s);
+  void offroadTransition(bool offroad);
+
+private slots:
+  void update();
+
+private:
+  QTimer *timer;
+  bool started_prev = true;
+};
+
+
+// device management class
+
+class Device : public QObject {
+  Q_OBJECT
+
+public:
+  Device(QObject *parent = 0);
+
+private:
+  // auto brightness
+  const float accel_samples = 5*UI_FREQ;
+
+  bool awake;
+  int awake_timeout = 0;
+  float accel_prev = 0;
+  float gyro_prev = 0;
+  float brightness_b = 0;
+  float brightness_m = 0;
+  float last_brightness = 0;
+  FirstOrderFilter brightness_filter;
+
+  QTimer *timer;
+
+  void updateBrightness(const UIState &s);
+  void updateWakefulness(const UIState &s);
+
+signals:
+  void displayPowerChanged(bool on);
+
+public slots:
+  void setAwake(bool on, bool reset);
+  void update(const UIState &s);
+};