Qt ui for PC (#2023)

* qt ui boilerplate

* this kinda works

* cleanup

* render inside other widget

* cleanup

* more cleanup

* Not needed

* Handle click

* Draw at 20Hz

* create paint.hpp

* move stuff around

* update sidebar

* Draw vision

* this works again

* add clickable settings button

* Only collapse sidebar when started

* always use qt on linux

* fix width

* scrollable area

* talk to NetworkManager

* code to add a connection

* params toggles

* small cleanup

* add qt5 to dockerfile

* Qt on mac

* Add qt to release files

* fix macos build

* nore more ifdefs needed

* add icons

* make a bit nicer

* Hide scrollbar

Co-authored-by: Comma Device <device@comma.ai>

Dockerfile.openpilot

@@ -38,6 +38,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     opencl-headers \
     python-dev \
     python-pip \
+    qt5-default \
     sudo \
     wget \
   && rm -rf /var/lib/apt/lists/*

SConstruct

@@ -156,6 +156,7 @@ env = Environment(
     "#selfdrive/camerad/include",
     "#selfdrive/loggerd/include",
     "#selfdrive/modeld",
+    "#selfdrive/ui",
     "#cereal/messaging",
     "#cereal",
     "#opendbc/can",
@@ -176,6 +177,52 @@ env = Environment(
   ]
 )
 
+qt_env = None
+if arch in ["x86_64", "Darwin", "larch64"]:
+  qt_env = env.Clone()
+
+  if arch == "larch64":
+    qt_env['QTDIR'] = "/usr/local/Qt-5.15.0"
+    QT_BASE = "/usr/local/Qt-5.15.0/"
+    qt_dirs = [
+      QT_BASE + "include/",
+      QT_BASE + "include/QtWidgets",
+      QT_BASE + "include/QtGui",
+      QT_BASE + "include/QtCore",
+      QT_BASE + "include/QtDBus",
+    ]
+    qt_env["RPATH"] += [QT_BASE + "lib"]
+  if arch == "Darwin":
+    qt_env['QTDIR'] = "/usr/local/opt/qt"
+    QT_BASE = "/usr/local/opt/qt/"
+    qt_dirs = [
+      QT_BASE + "include/",
+      QT_BASE + "include/QtWidgets",
+      QT_BASE + "include/QtGui",
+      QT_BASE + "include/QtCore",
+      QT_BASE + "include/QtDBus",
+    ]
+    qt_env["LINKFLAGS"] += ["-F" + QT_BASE + "lib"]
+  else:
+    qt_dirs = [
+      f"/usr/include/{arch}-linux-gnu/qt5",
+      f"/usr/include/{arch}-linux-gnu/qt5/QtWidgets",
+      f"/usr/include/{arch}-linux-gnu/qt5/QtGui",
+      f"/usr/include/{arch}-linux-gnu/qt5/QtCore",
+      f"/usr/include/{arch}-linux-gnu/qt5/QtDBus",
+    ]
+
+  qt_env.Tool('qt')
+  qt_env['CPPPATH'] += qt_dirs
+  qt_flags = [
+    "-D_REENTRANT",
+    "-DQT_NO_DEBUG",
+    "-DQT_WIDGETS_LIB",
+    "-DQT_GUI_LIB",
+    "-DQT_CORE_LIB"
+  ]
+  qt_env['CXXFLAGS'] += qt_flags
+
 if os.environ.get('SCONS_CACHE'):
   cache_dir = '/tmp/scons_cache'
 
@@ -214,7 +261,7 @@ def abspath(x):
 
 # still needed for apks
 zmq = 'zmq'
-Export('env', 'arch', 'zmq', 'SHARED', 'webcam', 'QCOM_REPLAY')
+Export('env', 'qt_env', 'arch', 'zmq', 'SHARED', 'webcam', 'QCOM_REPLAY')
 
 # cereal and messaging are shared with the system
 SConscript(['cereal/SConscript'])
@@ -262,6 +309,7 @@ SConscript(['selfdrive/loggerd/SConscript'])
 SConscript(['selfdrive/locationd/SConscript'])
 SConscript(['selfdrive/locationd/models/SConscript'])
 
+
 if arch == "aarch64":
   SConscript(['selfdrive/logcatd/SConscript'])
   SConscript(['selfdrive/sensord/SConscript'])

release/files_common

@@ -337,6 +337,9 @@ selfdrive/ui/text/Makefile
 selfdrive/ui/text/text
 selfdrive/ui/text/text.c
 
+selfdrive/ui/qt/*.cc
+selfdrive/ui/qt/*.hpp
+
 selfdrive/camerad/SConscript
 selfdrive/camerad/main.cc
 selfdrive/camerad/bufs.h

selfdrive/ui/.gitignore

@@ -0,0 +1 @@
+moc_*

selfdrive/ui/SConscript

@@ -1,17 +1,26 @@
-Import('env', 'arch', 'common', 'messaging', 'gpucommon', 'visionipc', 'cereal')
+Import('env', 'qt_env', 'arch', 'common', 'messaging', 'gpucommon', 'visionipc', 'cereal')
 
 src = ['ui.cc', 'paint.cc', 'sidebar.cc', '#phonelibs/nanovg/nanovg.c']
 libs = [common, 'zmq', 'czmq', 'capnp', 'kj', 'm', cereal, messaging, gpucommon, visionipc]
 
-if arch == "aarch64":
+
+if qt_env is None:
   src += ['sound.cc']
   libs += ['EGL', 'GLESv3', 'gnustl_shared', 'log', 'utils', 'gui', 'hardware', 'ui', 'CB', 'gsl', 'adreno_utils', 'OpenSLES', 'cutils', 'uuid', 'OpenCL']
   linkflags = ['-Wl,-rpath=/system/lib64,-rpath=/system/comma/usr/lib']
+
+  src = ["android_ui.cc"] + src
+  env.Program('_ui', src,
+    LINKFLAGS=linkflags,
+    LIBS=libs)
+
 else:
-  src += ['linux.cc']
+  qt_libs = ["GL", "pthread"]
-  libs += ['pthread', 'glfw']
+
-  linkflags = []
+  if arch == "Darwin":
+    qt_env["FRAMEWORKS"] += ["QtWidgets", "QtGui", "QtCore", "QtDBus"]
+  else:
+    qt_libs += ["Qt5Widgets", "Qt5Gui", "Qt5Core", "Qt5DBus"]
 
-env.Program('_ui', src,
+  qt_src = ["qt/ui.cc", "qt/window.cc", "qt/settings.cc"] + src
-  LINKFLAGS=linkflags,
+  qt_env.Program("_ui", qt_src, LIBS=qt_libs + libs)
-  LIBS=libs)

selfdrive/ui/android_ui.cc

@@ -0,0 +1,360 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/resource.h>
+#include <czmq.h>
+
+#include "common/util.h"
+#include "common/utilpp.h"
+#include "common/params.h"
+#include "common/touch.h"
+#include "common/timing.h"
+#include "common/swaglog.h"
+
+#include "ui.hpp"
+#include "paint.hpp"
+
+// Includes for light sensor
+#include <cutils/properties.h>
+#include <hardware/sensors.h>
+#include <utils/Timers.h>
+
+volatile sig_atomic_t do_exit = 0;
+static void set_do_exit(int sig) {
+  do_exit = 1;
+}
+
+
+static void* light_sensor_thread(void *args) {
+  set_thread_name("light_sensor");
+
+  int err;
+  UIState *s = (UIState*)args;
+  s->light_sensor = 0.0;
+
+  struct sensors_poll_device_t* device;
+  struct sensors_module_t* module;
+
+  hw_get_module(SENSORS_HARDWARE_MODULE_ID, (hw_module_t const**)&module);
+  sensors_open(&module->common, &device);
+
+  // need to do this
+  struct sensor_t const* list;
+  module->get_sensors_list(module, &list);
+
+  int SENSOR_LIGHT = 7;
+
+  err = device->activate(device, SENSOR_LIGHT, 0);
+  if (err != 0) goto fail;
+  err = device->activate(device, SENSOR_LIGHT, 1);
+  if (err != 0) goto fail;
+
+  device->setDelay(device, SENSOR_LIGHT, ms2ns(100));
+
+  while (!do_exit) {
+    static const size_t numEvents = 1;
+    sensors_event_t buffer[numEvents];
+
+    int n = device->poll(device, buffer, numEvents);
+    if (n < 0) {
+      LOG_100("light_sensor_poll failed: %d", n);
+    }
+    if (n > 0) {
+      s->light_sensor = buffer[0].light;
+    }
+  }
+  sensors_close(device);
+  return NULL;
+
+fail:
+  LOGE("LIGHT SENSOR IS MISSING");
+  s->light_sensor = 255;
+
+  return NULL;
+}
+
+static void ui_set_brightness(UIState *s, int brightness) {
+  static int last_brightness = -1;
+  if (last_brightness != brightness && (s->awake || brightness == 0)) {
+    if (set_brightness(brightness)) {
+      last_brightness = brightness;
+    }
+  }
+}
+
+int event_processing_enabled = -1;
+static void enable_event_processing(bool yes) {
+  if (event_processing_enabled != 1 && yes) {
+    system("service call window 18 i32 1");  // enable event processing
+    event_processing_enabled = 1;
+  } else if (event_processing_enabled != 0 && !yes) {
+    system("service call window 18 i32 0");  // disable event processing
+    event_processing_enabled = 0;
+  }
+}
+
+static void set_awake(UIState *s, bool awake) {
+  if (awake) {
+    // 30 second timeout
+    s->awake_timeout = 30*UI_FREQ;
+  }
+  if (s->awake != awake) {
+    s->awake = awake;
+
+    // TODO: replace command_awake and command_sleep with direct calls to android
+    if (awake) {
+      LOGW("awake normal");
+      framebuffer_set_power(s->fb, HWC_POWER_MODE_NORMAL);
+      enable_event_processing(true);
+    } else {
+      LOGW("awake off");
+      ui_set_brightness(s, 0);
+      framebuffer_set_power(s->fb, HWC_POWER_MODE_OFF);
+      enable_event_processing(false);
+    }
+  }
+}
+
+static void handle_vision_touch(UIState *s, int touch_x, int touch_y) {
+  if (s->started && (touch_x >= s->scene.ui_viz_rx - bdr_s)
+      && (s->active_app != cereal::UiLayoutState::App::SETTINGS)) {
+    if (!s->scene.frontview) {
+      s->scene.uilayout_sidebarcollapsed = !s->scene.uilayout_sidebarcollapsed;
+    } else {
+      write_db_value("IsDriverViewEnabled", "0", 1);
+    }
+  }
+}
+
+static void handle_sidebar_touch(UIState *s, int touch_x, int touch_y) {
+  if (!s->scene.uilayout_sidebarcollapsed && touch_x <= sbr_w) {
+    if (touch_x >= settings_btn_x && touch_x < (settings_btn_x + settings_btn_w)
+        && touch_y >= settings_btn_y && touch_y < (settings_btn_y + settings_btn_h)) {
+      s->active_app = cereal::UiLayoutState::App::SETTINGS;
+    }
+    else if (touch_x >= home_btn_x && touch_x < (home_btn_x + home_btn_w)
+             && touch_y >= home_btn_y && touch_y < (home_btn_y + home_btn_h)) {
+      if (s->started) {
+        s->active_app = cereal::UiLayoutState::App::NONE;
+        s->scene.uilayout_sidebarcollapsed = true;
+      } else {
+        s->active_app = cereal::UiLayoutState::App::HOME;
+      }
+    }
+  }
+}
+
+static void update_offroad_layout_state(UIState *s) {
+  static int timeout = 0;
+  static bool prev_collapsed = false;
+  static cereal::UiLayoutState::App prev_app = cereal::UiLayoutState::App::NONE;
+  if (timeout > 0) {
+    timeout--;
+  }
+  if (prev_collapsed != s->scene.uilayout_sidebarcollapsed || prev_app != s->active_app || timeout == 0) {
+    capnp::MallocMessageBuilder msg;
+    auto event = msg.initRoot<cereal::Event>();
+    event.setLogMonoTime(nanos_since_boot());
+    auto layout = event.initUiLayoutState();
+    layout.setActiveApp(s->active_app);
+    layout.setSidebarCollapsed(s->scene.uilayout_sidebarcollapsed);
+    s->pm->send("offroadLayout", msg);
+    LOGD("setting active app to %d with sidebar %d", (int)s->active_app, s->scene.uilayout_sidebarcollapsed);
+    prev_collapsed = s->scene.uilayout_sidebarcollapsed;
+    prev_app = s->active_app;
+    timeout = 2 * UI_FREQ;
+  }
+}
+
+int main(int argc, char* argv[]) {
+  int err;
+  setpriority(PRIO_PROCESS, 0, -14);
+
+  zsys_handler_set(NULL);
+  signal(SIGINT, (sighandler_t)set_do_exit);
+
+  UIState uistate = {};
+  UIState *s = &uistate;
+  ui_init(s);
+  set_awake(s, true);
+
+  enable_event_processing(true);
+
+  pthread_t connect_thread_handle;
+  err = pthread_create(&connect_thread_handle, NULL,
+                       vision_connect_thread, s);
+  assert(err == 0);
+
+  pthread_t light_sensor_thread_handle;
+  err = pthread_create(&light_sensor_thread_handle, NULL,
+                       light_sensor_thread, s);
+  assert(err == 0);
+
+  TouchState touch = {0};
+  touch_init(&touch);
+  s->touch_fd = touch.fd;
+
+  // light sensor scaling params
+  const bool LEON = util::read_file("/proc/cmdline").find("letv") != std::string::npos;
+
+  float brightness_b, brightness_m;
+  int result = read_param(&brightness_b, "BRIGHTNESS_B", true);
+  result += read_param(&brightness_m, "BRIGHTNESS_M", true);
+
+  if(result != 0){
+    brightness_b = LEON ? 10.0 : 5.0;
+    brightness_m = LEON ? 2.6 : 1.3;
+    write_param_float(brightness_b, "BRIGHTNESS_B", true);
+    write_param_float(brightness_m, "BRIGHTNESS_M", true);
+  }
+
+  float smooth_brightness = brightness_b;
+
+  const int MIN_VOLUME = LEON ? 12 : 9;
+  const int MAX_VOLUME = LEON ? 15 : 12;
+  assert(s->sound.init(MIN_VOLUME));
+
+  int draws = 0;
+
+  while (!do_exit) {
+    bool should_swap = false;
+    if (!s->started) {
+      // Delay a while to avoid 9% cpu usage while car is not started and user is keeping touching on the screen.
+      // Don't hold the lock while sleeping, so that vision_connect_thread have chances to get the lock.
+      usleep(30 * 1000);
+    }
+    pthread_mutex_lock(&s->lock);
+    double u1 = millis_since_boot();
+
+    // light sensor is only exposed on EONs
+    float clipped_brightness = (s->light_sensor*brightness_m) + brightness_b;
+    if (clipped_brightness > 512) clipped_brightness = 512;
+    smooth_brightness = clipped_brightness * 0.01 + smooth_brightness * 0.99;
+    if (smooth_brightness > 255) smooth_brightness = 255;
+    ui_set_brightness(s, (int)smooth_brightness);
+    ui_update_sizes(s);
+
+    // poll for touch events
+    int touch_x = -1, touch_y = -1;
+    int touched = touch_poll(&touch, &touch_x, &touch_y, 0);
+    if (touched == 1) {
+      set_awake(s, true);
+      handle_sidebar_touch(s, touch_x, touch_y);
+      handle_vision_touch(s, touch_x, touch_y);
+    }
+
+    if (!s->started) {
+      // always process events offroad
+      check_messages(s);
+
+      if (s->started) {
+        s->controls_timeout = 5 * UI_FREQ;
+      }
+    } else {
+      set_awake(s, true);
+      // Car started, fetch a new rgb image from ipc
+      if (s->vision_connected){
+        ui_update(s);
+      }
+
+      check_messages(s);
+
+      // Visiond process is just stopped, force a redraw to make screen blank again.
+      if (!s->started) {
+        s->scene.uilayout_sidebarcollapsed = false;
+        ui_draw(s);
+        glFinish();
+        should_swap = true;
+      }
+    }
+
+    // manage wakefulness
+    if (s->awake_timeout > 0) {
+      s->awake_timeout--;
+    } else {
+      set_awake(s, false);
+    }
+
+    // manage hardware disconnect
+    if (s->hardware_timeout > 0) {
+      s->hardware_timeout--;
+    } else {
+      s->scene.hwType = cereal::HealthData::HwType::UNKNOWN;
+    }
+
+    // Don't waste resources on drawing in case screen is off
+    if (s->awake) {
+      ui_draw(s);
+      glFinish();
+      should_swap = true;
+    }
+
+    s->sound.setVolume(fmin(MAX_VOLUME, MIN_VOLUME + s->scene.controls_state.getVEgo() / 5)); // up one notch every 5 m/s
+
+    if (s->controls_timeout > 0) {
+      s->controls_timeout--;
+    } else if (s->started && !s->scene.frontview) {
+      if (!s->controls_seen) {
+        // car is started, but controlsState hasn't been seen at all
+        s->scene.alert_text1 = "openpilot Unavailable";
+        s->scene.alert_text2 = "Waiting for controls to start";
+        s->scene.alert_size = cereal::ControlsState::AlertSize::MID;
+      } else {
+        // car is started, but controls is lagging or died
+        LOGE("Controls unresponsive");
+
+        if (s->scene.alert_text2 != "Controls Unresponsive") {
+          s->sound.play(AudibleAlert::CHIME_WARNING_REPEAT);
+        }
+
+        s->scene.alert_text1 = "TAKE CONTROL IMMEDIATELY";
+        s->scene.alert_text2 = "Controls Unresponsive";
+        s->scene.alert_size = cereal::ControlsState::AlertSize::FULL;
+        update_status(s, STATUS_ALERT);
+      }
+      ui_draw_vision_alert(s, s->scene.alert_size, s->status, s->scene.alert_text1.c_str(), s->scene.alert_text2.c_str());
+    }
+
+    read_param_timeout(&s->is_metric, "IsMetric", &s->is_metric_timeout);
+    read_param_timeout(&s->longitudinal_control, "LongitudinalControl", &s->longitudinal_control_timeout);
+    read_param_timeout(&s->limit_set_speed, "LimitSetSpeed", &s->limit_set_speed_timeout);
+    read_param_timeout(&s->speed_lim_off, "SpeedLimitOffset", &s->limit_set_speed_timeout);
+    int param_read = read_param_timeout(&s->last_athena_ping, "LastAthenaPingTime", &s->last_athena_ping_timeout);
+    if (param_read != -1) { // Param was updated this loop
+      if (param_read != 0) { // Failed to read param
+        s->scene.athenaStatus = NET_DISCONNECTED;
+      } else if (nanos_since_boot() - s->last_athena_ping < 70e9) {
+        s->scene.athenaStatus = NET_CONNECTED;
+      } else {
+        s->scene.athenaStatus = NET_ERROR;
+      }
+    }
+    update_offroad_layout_state(s);
+
+    pthread_mutex_unlock(&s->lock);
+
+    // the bg thread needs to be scheduled, so the main thread needs time without the lock
+    // safe to do this outside the lock?
+    if (should_swap) {
+      double u2 = millis_since_boot();
+      if (u2-u1 > 66) {
+        // warn on sub 15fps
+        LOGW("slow frame(%d) time: %.2f", draws, u2-u1);
+      }
+      draws++;
+      framebuffer_swap(s->fb);
+    }
+  }
+
+  set_awake(s, true);
+
+  // wake up bg thread to exit
+  pthread_mutex_lock(&s->lock);
+  pthread_mutex_unlock(&s->lock);
+
+  // join light_sensor_thread?
+  err = pthread_join(connect_thread_handle, NULL);
+  assert(err == 0);
+  delete s->sm;
+  delete s->pm;
+  return 0;
+}

selfdrive/ui/linux.cc

@@ -1,107 +0,0 @@
-#include <stdio.h>
-#include <stdint.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <string.h>
-
-#include "ui.hpp"
-
-#ifndef __APPLE__
-#define GLFW_INCLUDE_ES2
-#else
-#define GLFW_INCLUDE_GLCOREARB
-#endif
-
-#define GLFW_INCLUDE_GLEXT
-#include <GLFW/glfw3.h>
-
-typedef struct FramebufferState FramebufferState;
-typedef struct TouchState TouchState;
-
-extern "C" {
-
-FramebufferState* framebuffer_init(
-    const char* name, int32_t layer, int alpha,
-    int *out_w, int *out_h) {
-  glfwInit();
-
-#ifndef __APPLE__
-  glfwWindowHint(GLFW_CLIENT_API, GLFW_OPENGL_ES_API);
-  glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3);
-  glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 0);
-#else
-  glfwWindowHint(GLFW_OPENGL_FORWARD_COMPAT, GL_TRUE);
-  glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE);
-  glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3);
-  glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 2);
-#endif
-  glfwWindowHint(GLFW_RESIZABLE, 0);
-  GLFWwindow* window;
-  window = glfwCreateWindow(1920, 1080, "ui", NULL, NULL);
-  if (!window) {
-    printf("glfwCreateWindow failed\n");
-  }
-
-  glfwMakeContextCurrent(window);
-  glfwSwapInterval(1);
-
-  // clear screen
-  glClearColor(0.2f, 0.2f, 0.2f, 1.0f);
-  glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
-  framebuffer_swap((FramebufferState*)window);
-
-  if (out_w) *out_w = 1920;
-  if (out_h) *out_h = 1080;
-
-  return (FramebufferState*)window;
-}
-
-void framebuffer_set_power(FramebufferState *s, int mode) {
-}
-
-void framebuffer_swap(FramebufferState *s) {
-  glfwSwapBuffers((GLFWwindow*)s);
-  glfwPollEvents();
-}
-
-bool set_brightness(int brightness) { return true; }
-
-void touch_init(TouchState *s) {
-  printf("touch_init\n");
-}
-
-int touch_poll(TouchState *s, int* out_x, int* out_y, int timeout) {
-  return -1;
-}
-
-int touch_read(TouchState *s, int* out_x, int* out_y) {
-  return -1;
-}
-
-}
-
-#include "sound.hpp"
-
-bool Sound::init(int volume) { return true; }
-bool Sound::play(AudibleAlert alert) { printf("play sound: %d\n", (int)alert); return true; }
-void Sound::stop() {}
-void Sound::setVolume(int volume) {}
-Sound::~Sound() {}
-
-#include "common/visionimg.h"
-#include <sys/mman.h>
-
-GLuint visionimg_to_gl(const VisionImg *img, EGLImageKHR *pkhr, void **pph) {
-  unsigned int texture;
-  glGenTextures(1, &texture);
-  glBindTexture(GL_TEXTURE_2D, texture);
-  glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, img->width, img->height, 0, GL_RGB, GL_UNSIGNED_BYTE, *pph);
-  glGenerateMipmap(GL_TEXTURE_2D);
-  *pkhr = (EGLImageKHR)1; // not NULL
-  return texture;
-}
-
-void visionimg_destroy_gl(EGLImageKHR khr, void *ph) {
-  // empty
-}
-

selfdrive/ui/paint.cc

@@ -2,10 +2,10 @@
 #include <assert.h>
 #include <map>
 #include <cmath>
+#include <iostream>
 #include "common/util.h"
 
 #define NANOVG_GLES3_IMPLEMENTATION
-
 #include "nanovg_gl.h"
 #include "nanovg_gl_utils.h"
 
@@ -13,6 +13,9 @@ extern "C"{
 #include "common/glutil.h"
 }
 
+#include "paint.hpp"
+#include "sidebar.hpp"
+
 // TODO: this is also hardcoded in common/transformations/camera.py
 const mat3 intrinsic_matrix = (mat3){{
   910., 0., 582.,
@@ -348,7 +351,7 @@ static void ui_draw_world(UIState *s) {
     return;
   }
 
-  const int inner_height = viz_w*9/16;
+  const int inner_height = float(viz_w) * vwp_h / vwp_w;
   const int ui_viz_rx = scene->ui_viz_rx;
   const int ui_viz_rw = scene->ui_viz_rw;
   const int ui_viz_ro = scene->ui_viz_ro;
@@ -358,10 +361,13 @@ static void ui_draw_world(UIState *s) {
 
   nvgTranslate(s->vg, ui_viz_rx+ui_viz_ro, box_y + (box_h-inner_height)/2.0);
   nvgScale(s->vg, (float)viz_w / s->fb_w, (float)inner_height / s->fb_h);
-  nvgTranslate(s->vg, 240.0f, 0.0);
+
-  nvgTranslate(s->vg, -1440.0f / 2, -1080.0f / 2);
+  float w = 1440.0f; // Why 1440?
+  nvgTranslate(s->vg, (vwp_w - w) / 2.0f, 0.0);
+
+  nvgTranslate(s->vg, -w / 2, -1080.0f / 2);
   nvgScale(s->vg, 2.0, 2.0);
-  nvgScale(s->vg, 1440.0f / s->rgb_width, 1080.0f / s->rgb_height);
+  nvgScale(s->vg, w / s->rgb_width, 1080.0f / s->rgb_height);
 
   // Draw lane edges and vision/mpc tracks
   ui_draw_vision_lanes(s);

selfdrive/ui/paint.hpp

@@ -0,0 +1,11 @@
+#pragma once
+#include "ui.hpp"
+
+
+void ui_draw_vision_alert(UIState *s, cereal::ControlsState::AlertSize va_size, int va_color,
+                          const char* va_text1, const char* va_text2);
+void ui_draw(UIState *s);
+void ui_draw_image(NVGcontext *vg, float x, float y, float w, float h, int image, float alpha);
+void ui_draw_rect(NVGcontext *vg, float x, float y, float w, float h, NVGcolor color, float r = 0, int width = 0);
+void ui_draw_rect(NVGcontext *vg, float x, float y, float w, float h, NVGpaint &paint, float r = 0);
+void ui_nvg_init(UIState *s);

selfdrive/ui/qt/settings.cc

@@ -0,0 +1,140 @@
+#include <string>
+#include <iostream>
+#include <sstream>
+#include <cassert>
+
+#include "qt/settings.hpp"
+
+#include <QVBoxLayout>
+#include <QHBoxLayout>
+#include <QPushButton>
+#include <QLabel>
+#include <QScrollArea>
+#include <QScroller>
+#include <QScrollerProperties>
+#include <QDebug>
+#include <QPixmap>
+
+#include "common/params.h"
+
+ParamsToggle::ParamsToggle(QString param, QString title, QString description, QString icon, QWidget *parent): QFrame(parent) , param(param) {
+  QHBoxLayout *hlayout = new QHBoxLayout;
+  QVBoxLayout *vlayout = new QVBoxLayout;
+
+  hlayout->addSpacing(25);
+  if (icon.length()){
+    QPixmap pix(icon);
+    QLabel *icon = new QLabel();
+    icon->setPixmap(pix.scaledToWidth(100, Qt::SmoothTransformation));
+    icon->setSizePolicy(QSizePolicy(QSizePolicy::Fixed, QSizePolicy::Fixed));
+    hlayout->addWidget(icon);
+  } else{
+    hlayout->addSpacing(100);
+  }
+  hlayout->addSpacing(25);
+
+  checkbox = new QCheckBox(title);
+  QLabel *label = new QLabel(description);
+  label->setWordWrap(true);
+
+  vlayout->addWidget(checkbox);
+  vlayout->addWidget(label);
+  hlayout->addLayout(vlayout);
+
+  setLayout(hlayout);
+
+  auto p = read_db_bytes(param.toStdString().c_str());
+  if (p.size()){
+    checkbox->setChecked(p[0] == '1');
+  }
+
+  setStyleSheet(R"(
+    QCheckBox { font-size: 40px }
+    QLabel { font-size: 20px }
+    * {
+      background-color: #114265;
+    }
+  )");
+
+  QObject::connect(checkbox, SIGNAL(stateChanged(int)), this, SLOT(checkboxClicked(int)));
+}
+
+void ParamsToggle::checkboxClicked(int state){
+  char value = state ? '1': '0';
+  write_db_value(param.toStdString().c_str(), &value, 1);
+}
+
+SettingsWindow::SettingsWindow(QWidget *parent) : QWidget(parent) {
+  QWidget *container = new QWidget(this);
+
+  QVBoxLayout *settings_list = new QVBoxLayout();
+  settings_list->addWidget(new ParamsToggle("OpenpilotEnabledToggle",
+                                            "Enable Openpilot",
+                                            "Use the openpilot system for adaptive cruise control and lane keep driver assistance. Your attention is required at all times to use this feature. Changing this setting takes effect when the car is powered off.",
+                                            "../assets/offroad/icon_openpilot.png"
+                                              ));
+  settings_list->addWidget(new ParamsToggle("LaneChangeEnabled",
+                                            "Enable Lane Change Assist",
+                                            "Perform assisted lane changes with openpilot by checking your surroundings for safety, activating the turn signal and gently nudging the steering wheel towards your desired lane. openpilot is not capable of checking if a lane change is safe. You must continuously observe your surroundings to use this feature.",
+                                            "../assets/offroad/icon_road.png"
+                                              ));
+  settings_list->addWidget(new ParamsToggle("IsLdwEnabled",
+                                            "Enable Lane Departure Warnings",
+                                            "Receive alerts to steer back into the lane when your vehicle drifts over a detected lane line without a turn signal activated while driving over 31mph (50kph).",
+                                            "../assets/offroad/icon_warning.png"
+                                              ));
+  settings_list->addWidget(new ParamsToggle("RecordFront",
+                                            "Record and Upload Driver Camera",
+                                            "Upload data from the driver facing camera and help improve the driver monitoring algorithm.",
+                                            "../assets/offroad/icon_network.png"
+                                            ));
+  settings_list->addWidget(new ParamsToggle("IsRHD",
+                                            "Enable Right-Hand Drive",
+                                            "Allow openpilot to obey left-hand traffic conventions and perform driver monitoring on right driver seat.",
+                                            "../assets/offroad/icon_openpilot_mirrored.png"
+                                            ));
+  settings_list->addWidget(new ParamsToggle("IsMetric",
+                                            "Use Metric System",
+                                            "Display speed in km/h instead of mp/h.",
+                                            "../assets/offroad/icon_metric.png"
+                                            ));
+  settings_list->addWidget(new ParamsToggle("CommunityFeaturesToggle",
+                                            "Enable Community Features",
+                                            "Use features from the open source community that are not maintained or supported by comma.ai and have not been confirmed to meet the standard safety model. These features include community supported cars and community supported hardware. Be extra cautious when using these features",
+                                            "../assets/offroad/icon_shell.png"
+                                            ));
+
+  settings_list->setSpacing(25);
+
+  container->setLayout(settings_list);
+  container->setFixedWidth(1650);
+
+  QScrollArea *scrollArea = new QScrollArea;
+  scrollArea->setHorizontalScrollBarPolicy(Qt::ScrollBarAlwaysOff);
+  scrollArea->setVerticalScrollBarPolicy(Qt::ScrollBarAlwaysOff);
+  scrollArea->setWidget(container);
+
+  QScrollerProperties sp;
+  sp.setScrollMetric(QScrollerProperties::DecelerationFactor, 2.0);
+
+  QScroller* qs = QScroller::scroller(scrollArea);
+  qs->setScrollerProperties(sp);
+
+  QHBoxLayout *main_layout = new QHBoxLayout;
+  main_layout->addSpacing(50);
+  main_layout->addWidget(scrollArea);
+
+  QPushButton * button = new QPushButton("Close");
+  main_layout->addWidget(button);
+  main_layout->addSpacing(20);
+
+  setLayout(main_layout);
+
+  QScroller::grabGesture(scrollArea, QScroller::LeftMouseButtonGesture);
+  QObject::connect(button, SIGNAL(clicked()), this, SIGNAL(closeSettings()));
+
+  setStyleSheet(R"(
+    QPushButton { font-size: 40px }
+  )");
+
+}

selfdrive/ui/qt/settings.hpp

@@ -0,0 +1,28 @@
+#pragma once
+
+#include <QWidget>
+#include <QFrame>
+#include <QTimer>
+#include <QCheckBox>
+
+class ParamsToggle : public QFrame {
+  Q_OBJECT
+
+private:
+  QCheckBox *checkbox;
+  QString param;
+public:
+  explicit ParamsToggle(QString param, QString title, QString description, QString icon, QWidget *parent = 0);
+public slots:
+  void checkboxClicked(int state);
+};
+
+
+
+class SettingsWindow : public QWidget {
+  Q_OBJECT
+public:
+  explicit SettingsWindow(QWidget *parent = 0);
+signals:
+  void closeSettings();
+};

selfdrive/ui/qt/ui.cc

@@ -0,0 +1,15 @@
+#include <QApplication>
+
+#include "window.hpp"
+
+int main(int argc, char *argv[])
+{
+  QApplication a(argc, argv);
+
+  MainWindow w;
+
+  w.setFixedSize(vwp_w, vwp_h);
+  w.show();
+
+  return a.exec();
+}

selfdrive/ui/qt/wifi.cc

@@ -0,0 +1,69 @@
+#include <QtDBus>
+#include <QDebug>
+
+typedef QMap<QString, QMap<QString, QVariant> > Connection;
+Q_DECLARE_METATYPE(Connection)
+
+void wifi_stuff(){
+  qDBusRegisterMetaType<Connection>();
+
+  QString nm_path = "/org/freedesktop/NetworkManager";
+  QString nm_settings_path = "/org/freedesktop/NetworkManager/Settings";
+
+  QString nm_iface = "org.freedesktop.NetworkManager";
+  QString props_iface = "org.freedesktop.DBus.Properties";
+  QString nm_settings_iface = "org.freedesktop.NetworkManager.Settings";
+
+  QString nm_service = "org.freedesktop.NetworkManager";
+  QString device_service = "org.freedesktop.NetworkManager.Device";
+
+  QDBusConnection bus = QDBusConnection::systemBus();
+
+  // Get devices
+  QDBusInterface nm(nm_service, nm_path, nm_iface, bus);
+  QDBusMessage response = nm.call("GetDevices");
+  QVariant first =  response.arguments().at(0);
+
+  const QDBusArgument &args = first.value<QDBusArgument>();
+  args.beginArray();
+  while (!args.atEnd()) {
+    QDBusObjectPath path;
+    args >> path;
+
+    // Get device type
+    QDBusInterface device_props(nm_service, path.path(), props_iface, bus);
+    QDBusMessage response = device_props.call("Get", device_service, "DeviceType");
+    QVariant first =  response.arguments().at(0);
+    QDBusVariant dbvFirst = first.value<QDBusVariant>();
+    QVariant vFirst = dbvFirst.variant();
+    uint device_type = vFirst.value<uint>();
+    qDebug() << path.path() << device_type;
+  }
+  args.endArray();
+
+
+  // Add connection
+  Connection connection;
+  connection["connection"]["type"] = "802-11-wireless";
+  connection["connection"]["uuid"] = QUuid::createUuid().toString().remove('{').remove('}');
+  connection["connection"]["id"] = "Connection 1";
+
+  connection["802-11-wireless"]["ssid"] = QByteArray("<ssid>");
+  connection["802-11-wireless"]["mode"] = "infrastructure";
+
+  connection["802-11-wireless-security"]["key-mgmt"] = "wpa-psk";
+  connection["802-11-wireless-security"]["auth-alg"] = "open";
+  connection["802-11-wireless-security"]["psk"] = "<password>";
+
+  connection["ipv4"]["method"] = "auto";
+  connection["ipv6"]["method"] = "ignore";
+
+
+  QDBusInterface nm_settings(nm_service, nm_settings_path, nm_settings_iface, bus);
+  QDBusReply<QDBusObjectPath> result = nm_settings.call("AddConnection", QVariant::fromValue(connection));
+  if (!result.isValid()) {
+    qDebug() << result.error().name() << result.error().message();
+  } else {
+    qDebug() << result.value().path();
+  }
+}

selfdrive/ui/qt/window.cc

@@ -0,0 +1,143 @@
+#include <cassert>
+#include <iostream>
+#include <cmath>
+#include <iostream>
+#include <signal.h>
+
+#include <QVBoxLayout>
+#include <QMouseEvent>
+#include <QPushButton>
+#include <QGridLayout>
+
+#include "window.hpp"
+#include "settings.hpp"
+
+#include "paint.hpp"
+#include "sound.hpp"
+
+volatile sig_atomic_t do_exit = 0;
+
+MainWindow::MainWindow(QWidget *parent) : QWidget(parent) {
+  main_layout = new QStackedLayout;
+
+  GLWindow * glWindow = new GLWindow(this);
+  main_layout->addWidget(glWindow);
+
+  SettingsWindow * settingsWindow = new SettingsWindow(this);
+  main_layout->addWidget(settingsWindow);
+
+
+  main_layout->setMargin(0);
+  setLayout(main_layout);
+  QObject::connect(glWindow, SIGNAL(openSettings()), this, SLOT(openSettings()));
+  QObject::connect(settingsWindow, SIGNAL(closeSettings()), this, SLOT(closeSettings()));
+
+  setStyleSheet(R"(
+    * {
+      color: white;
+      background-color: #072339;
+    }
+  )");
+}
+
+void MainWindow::openSettings(){
+  main_layout->setCurrentIndex(1);
+}
+
+void MainWindow::closeSettings(){
+  main_layout->setCurrentIndex(0);
+}
+
+
+
+GLWindow::GLWindow(QWidget *parent) : QOpenGLWidget(parent) {
+  timer = new QTimer(this);
+  QObject::connect(timer, SIGNAL(timeout()), this, SLOT(timerUpdate()));
+}
+
+GLWindow::~GLWindow() {
+  makeCurrent();
+  doneCurrent();
+}
+
+void GLWindow::initializeGL() {
+  initializeOpenGLFunctions();
+
+  ui_state = new UIState();
+  ui_init(ui_state);
+  ui_state->fb_w = vwp_w;
+  ui_state->fb_h = vwp_h;
+
+  int err = pthread_create(&connect_thread_handle, NULL,
+                           vision_connect_thread, ui_state);
+  assert(err == 0);
+
+  timer->start(50);
+}
+
+void GLWindow::timerUpdate(){
+  pthread_mutex_lock(&ui_state->lock);
+
+  ui_update_sizes(ui_state);
+
+  check_messages(ui_state);
+  if (ui_state->vision_connected){
+    ui_update(ui_state);
+  }
+  pthread_mutex_unlock(&ui_state->lock);
+
+  update();
+}
+
+void GLWindow::resizeGL(int w, int h) {
+  std::cout << "resize " << w << "x" << h << std::endl;
+}
+
+void GLWindow::paintGL() {
+  pthread_mutex_lock(&ui_state->lock);
+  ui_draw(ui_state);
+  pthread_mutex_unlock(&ui_state->lock);
+}
+
+void GLWindow::mousePressEvent(QMouseEvent *e) {
+  // Settings button click
+  if (!ui_state->scene.uilayout_sidebarcollapsed && e->x() <= sbr_w) {
+    if (e->x() >= settings_btn_x && e->x() < (settings_btn_x + settings_btn_w)
+        && e->y() >= settings_btn_y && e->y() < (settings_btn_y + settings_btn_h)) {
+      emit openSettings();
+    }
+  }
+
+  // Vision click
+  if (ui_state->started && (e->x() >= ui_state->scene.ui_viz_rx - bdr_s)){
+    ui_state->scene.uilayout_sidebarcollapsed = !ui_state->scene.uilayout_sidebarcollapsed;
+  }
+
+}
+
+
+/* HACKS */
+bool Sound::init(int volume) { return true; }
+bool Sound::play(AudibleAlert alert) { printf("play sound: %d\n", (int)alert); return true; }
+void Sound::stop() {}
+void Sound::setVolume(int volume) {}
+Sound::~Sound() {}
+
+GLuint visionimg_to_gl(const VisionImg *img, EGLImageKHR *pkhr, void **pph) {
+  unsigned int texture;
+  glGenTextures(1, &texture);
+  glBindTexture(GL_TEXTURE_2D, texture);
+  glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, img->width, img->height, 0, GL_RGB, GL_UNSIGNED_BYTE, *pph);
+  glGenerateMipmap(GL_TEXTURE_2D);
+  *pkhr = (EGLImageKHR)1; // not NULL
+  return texture;
+}
+
+void visionimg_destroy_gl(EGLImageKHR khr, void *ph) {
+  // empty
+}
+
+FramebufferState* framebuffer_init(const char* name, int32_t layer, int alpha,
+                                   int *out_w, int *out_h) {
+  return (FramebufferState*)1; // not null
+}

selfdrive/ui/qt/window.hpp

@@ -0,0 +1,55 @@
+#pragma once
+
+#include <QWidget>
+#include <QTimer>
+#include <QGuiApplication>
+#include <QOpenGLWidget>
+#include <QOpenGLFunctions>
+#include <QStackedLayout>
+
+#include "ui/ui.hpp"
+
+class MainWindow : public QWidget
+{
+  Q_OBJECT
+
+public:
+  explicit MainWindow(QWidget *parent = 0);
+
+private:
+  QStackedLayout *main_layout;
+
+public slots:
+  void openSettings();
+  void closeSettings();
+
+};
+
+
+class GLWindow : public QOpenGLWidget, protected QOpenGLFunctions
+{
+  Q_OBJECT
+
+public:
+  using QOpenGLWidget::QOpenGLWidget;
+  explicit GLWindow(QWidget *parent = 0);
+  ~GLWindow();
+
+protected:
+  void mousePressEvent(QMouseEvent *e) override;
+  void initializeGL() override;
+  void resizeGL(int w, int h) override;
+  void paintGL() override;
+
+
+private:
+  QTimer * timer;
+  UIState * ui_state;
+  pthread_t connect_thread_handle;
+
+public slots:
+  void timerUpdate();
+
+signals:
+  void openSettings();
+};

selfdrive/ui/sidebar.cc

@@ -2,7 +2,9 @@
 #include <string.h>
 #include <math.h>
 #include <map>
-#include "ui.hpp"
+
+#include "paint.hpp"
+#include "sidebar.hpp"
 
 static void ui_draw_sidebar_background(UIState *s) {
   int sbr_x = !s->scene.uilayout_sidebarcollapsed ? 0 : -(sbr_w) + bdr_s * 2;

selfdrive/ui/sidebar.hpp

@@ -0,0 +1,4 @@
+#pragma once
+#include "ui.hpp"
+
+void ui_draw_sidebar(UIState *s);

selfdrive/ui/ui.cc

@@ -4,168 +4,25 @@
 #include <unistd.h>
 #include <assert.h>
 #include <sys/mman.h>
-#include <string>
-#include <sstream>
-#include <sys/resource.h>
 #include <czmq.h>
+
 #include "common/util.h"
-#include "common/timing.h"
 #include "common/swaglog.h"
-#include "common/touch.h"
 #include "common/visionimg.h"
-#include "common/params.h"
 #include "common/utilpp.h"
 #include "ui.hpp"
+#include "paint.hpp"
 
-static void ui_set_brightness(UIState *s, int brightness) {
+extern volatile sig_atomic_t do_exit;
-  static int last_brightness = -1;
-  if (last_brightness != brightness && (s->awake || brightness == 0)) {
-    if (set_brightness(brightness)) {
-      last_brightness = brightness;
-    }
-  }
-}
 
-int event_processing_enabled = -1;
-static void enable_event_processing(bool yes) {
-  if (event_processing_enabled != 1 && yes) {
-    system("service call window 18 i32 1");  // enable event processing
-    event_processing_enabled = 1;
-  } else if (event_processing_enabled != 0 && !yes) {
-    system("service call window 18 i32 0");  // disable event processing
-    event_processing_enabled = 0;
-  }
-}
 
-static void set_awake(UIState *s, bool awake) {
+int write_param_float(float param, const char* param_name, bool persistent_param) {
-#ifdef QCOM
-  if (awake) {
-    // 30 second timeout
-    s->awake_timeout = 30*UI_FREQ;
-  }
-  if (s->awake != awake) {
-    s->awake = awake;
-
-    // TODO: replace command_awake and command_sleep with direct calls to android
-    if (awake) {
-      LOGW("awake normal");
-      framebuffer_set_power(s->fb, HWC_POWER_MODE_NORMAL);
-      enable_event_processing(true);
-    } else {
-      LOGW("awake off");
-      ui_set_brightness(s, 0);
-      framebuffer_set_power(s->fb, HWC_POWER_MODE_OFF);
-      enable_event_processing(false);
-    }
-  }
-#else
-  // computer UI doesn't sleep
-  s->awake = true;
-#endif
-}
-
-static void update_offroad_layout_state(UIState *s) {
-#ifdef QCOM
-  static int timeout = 0;
-  static bool prev_collapsed = false;
-  static cereal::UiLayoutState::App prev_app = cereal::UiLayoutState::App::NONE;
-  if (timeout > 0) {
-    timeout--;
-  }
-  if (prev_collapsed != s->scene.uilayout_sidebarcollapsed || prev_app != s->active_app || timeout == 0) {
-    capnp::MallocMessageBuilder msg;
-    auto event = msg.initRoot<cereal::Event>();
-    event.setLogMonoTime(nanos_since_boot());
-    auto layout = event.initUiLayoutState();
-    layout.setActiveApp(s->active_app);
-    layout.setSidebarCollapsed(s->scene.uilayout_sidebarcollapsed);
-    s->pm->send("offroadLayout", msg);
-    LOGD("setting active app to %d with sidebar %d", (int)s->active_app, s->scene.uilayout_sidebarcollapsed);
-    prev_collapsed = s->scene.uilayout_sidebarcollapsed;
-    prev_app = s->active_app;
-    timeout = 2 * UI_FREQ;
-  }
-#endif
-}
-
-static void handle_sidebar_touch(UIState *s, int touch_x, int touch_y) {
-  if (!s->scene.uilayout_sidebarcollapsed && touch_x <= sbr_w) {
-    if (touch_x >= settings_btn_x && touch_x < (settings_btn_x + settings_btn_w)
-      && touch_y >= settings_btn_y && touch_y < (settings_btn_y + settings_btn_h)) {
-      s->active_app = cereal::UiLayoutState::App::SETTINGS;
-    }
-    else if (touch_x >= home_btn_x && touch_x < (home_btn_x + home_btn_w)
-      && touch_y >= home_btn_y && touch_y < (home_btn_y + home_btn_h)) {
-      if (s->started) {
-        s->active_app = cereal::UiLayoutState::App::NONE;
-        s->scene.uilayout_sidebarcollapsed = true;
-      } else {
-        s->active_app = cereal::UiLayoutState::App::HOME;
-      }
-    }
-  }
-}
-
-static void handle_vision_touch(UIState *s, int touch_x, int touch_y) {
-  if (s->started && (touch_x >= s->scene.ui_viz_rx - bdr_s)
-    && (s->active_app != cereal::UiLayoutState::App::SETTINGS)) {
-    if (!s->scene.frontview) {
-      s->scene.uilayout_sidebarcollapsed = !s->scene.uilayout_sidebarcollapsed;
-    } else {
-      write_db_value("IsDriverViewEnabled", "0", 1);
-    }
-  }
-}
-
-volatile sig_atomic_t do_exit = 0;
-static void set_do_exit(int sig) {
-  do_exit = 1;
-}
-
-template <class T>
-static int read_param(T* param, const char *param_name, bool persistent_param = false){
-  T param_orig = *param;
-  char *value;
-  size_t sz;
-
-  int result = read_db_value(param_name, &value, &sz, persistent_param);
-  if (result == 0){
-    std::string s = std::string(value, sz); // value is not null terminated
-    free(value);
-
-    // Parse result
-    std::istringstream iss(s);
-    iss >> *param;
-
-    // Restore original value if parsing failed
-    if (iss.fail()) {
-      *param = param_orig;
-      result = -1;
-    }
-  }
-  return result;
-}
-
-template <class T>
-static int read_param_timeout(T* param, const char* param_name, int* timeout, bool persistent_param = false) {
-  int result = -1;
-  if (*timeout > 0){
-    (*timeout)--;
-  } else {
-    *timeout = 2 * UI_FREQ; // 0.5Hz
-    result = read_param(param, param_name, persistent_param);
-  }
-  return result;
-}
-
-static int write_param_float(float param, const char* param_name, bool persistent_param = false) {
   char s[16];
   int size = snprintf(s, sizeof(s), "%f", param);
   return write_db_value(param_name, s, MIN(size, sizeof(s)), persistent_param);
 }
 
-static void ui_init(UIState *s) {
+void ui_init(UIState *s) {
-
   pthread_mutex_init(&s->lock, NULL);
   s->sm = new SubMaster({"model", "controlsState", "uiLayoutState", "liveCalibration", "radarState", "thermal",
                          "health", "ubloxGnss", "driverState", "dMonitoringState"
@@ -180,12 +37,9 @@ static void ui_init(UIState *s) {
   s->started = false;
   s->vision_seen = false;
 
-  // init display
   s->fb = framebuffer_init("ui", 0, true, &s->fb_w, &s->fb_h);
   assert(s->fb);
 
-  set_awake(s, true);
-
   ui_nvg_init(s);
 }
 
@@ -227,7 +81,7 @@ static void ui_init_vision(UIState *s, const VisionStreamBufs back_bufs,
   s->limit_set_speed_timeout = UI_FREQ;
 }
 
-static void update_status(UIState *s, int status) {
+void update_status(UIState *s, int status) {
   if (s->status != status) {
     s->status = status;
   }
@@ -373,13 +227,21 @@ void handle_message(UIState *s, SubMaster &sm) {
   }
 }
 
-static void check_messages(UIState *s) {
+void check_messages(UIState *s) {
   if (s->sm->update(0) > 0){
     handle_message(s, *(s->sm));
   }
 }
 
-static void ui_update(UIState *s) {
+void ui_update_sizes(UIState *s){
+  // resize vision for collapsing sidebar
+  const bool hasSidebar = !s->scene.uilayout_sidebarcollapsed;
+  s->scene.ui_viz_rx = hasSidebar ? box_x : (box_x - sbr_w + (bdr_s * 2));
+  s->scene.ui_viz_rw = hasSidebar ? box_w : (box_w + sbr_w - (bdr_s * 2));
+  s->scene.ui_viz_ro = hasSidebar ? -(sbr_w - 6 * bdr_s) : 0;
+}
+
+void ui_update(UIState *s) {
   int err;
 
   if (s->vision_connect_firstrun) {
@@ -573,7 +435,7 @@ static int vision_subscribe(int fd, VisionPacket *rp, VisionStreamType type) {
   return 1;
 }
 
-static void* vision_connect_thread(void *args) {
+void* vision_connect_thread(void *args) {
   set_thread_name("vision_connect");
 
   UIState *s = (UIState*)args;
@@ -610,261 +472,3 @@ static void* vision_connect_thread(void *args) {
   }
   return NULL;
 }
-
-#ifdef QCOM
-
-#include <cutils/properties.h>
-#include <hardware/sensors.h>
-#include <utils/Timers.h>
-
-static void* light_sensor_thread(void *args) {
-  int err;
-  set_thread_name("light_sensor");
-
-  UIState *s = (UIState*)args;
-  s->light_sensor = 0.0;
-
-  struct sensors_poll_device_t* device;
-  struct sensors_module_t* module;
-
-  hw_get_module(SENSORS_HARDWARE_MODULE_ID, (hw_module_t const**)&module);
-  sensors_open(&module->common, &device);
-
-  // need to do this
-  struct sensor_t const* list;
-  module->get_sensors_list(module, &list);
-
-  int SENSOR_LIGHT = 7;
-
-  err = device->activate(device, SENSOR_LIGHT, 0);
-  if (err != 0) goto fail;
-  err = device->activate(device, SENSOR_LIGHT, 1);
-  if (err != 0) goto fail;
-
-  device->setDelay(device, SENSOR_LIGHT, ms2ns(100));
-
-  while (!do_exit) {
-    static const size_t numEvents = 1;
-    sensors_event_t buffer[numEvents];
-
-    int n = device->poll(device, buffer, numEvents);
-    if (n < 0) {
-      LOG_100("light_sensor_poll failed: %d", n);
-    }
-    if (n > 0) {
-      s->light_sensor = buffer[0].light;
-    }
-  }
-  sensors_close(device);
-  return NULL;
-
-fail:
-  LOGE("LIGHT SENSOR IS MISSING");
-  s->light_sensor = 255;
-  return NULL;
-}
-
-#endif
-
-int main(int argc, char* argv[]) {
-  int err;
-  setpriority(PRIO_PROCESS, 0, -14);
-
-  zsys_handler_set(NULL);
-  signal(SIGINT, (sighandler_t)set_do_exit);
-
-  UIState uistate = {};
-  UIState *s = &uistate;
-  ui_init(s);
-
-  enable_event_processing(true);
-
-  pthread_t connect_thread_handle;
-  err = pthread_create(&connect_thread_handle, NULL,
-                       vision_connect_thread, s);
-  assert(err == 0);
-
-#ifdef QCOM
-  pthread_t light_sensor_thread_handle;
-  err = pthread_create(&light_sensor_thread_handle, NULL,
-                       light_sensor_thread, s);
-  assert(err == 0);
-#endif
-
-  TouchState touch = {0};
-  touch_init(&touch);
-  s->touch_fd = touch.fd;
-
-  // light sensor scaling params
-  const bool LEON = util::read_file("/proc/cmdline").find("letv") != std::string::npos;
-
-  float brightness_b, brightness_m;
-  int result = read_param(&brightness_b, "BRIGHTNESS_B", true);
-  result += read_param(&brightness_m, "BRIGHTNESS_M", true);
-
-  if(result != 0){
-    brightness_b = LEON ? 10.0 : 5.0;
-    brightness_m = LEON ? 2.6 : 1.3;
-    write_param_float(brightness_b, "BRIGHTNESS_B", true);
-    write_param_float(brightness_m, "BRIGHTNESS_M", true);
-  }
-
-  float smooth_brightness = brightness_b;
-
-  const int MIN_VOLUME = LEON ? 12 : 9;
-  const int MAX_VOLUME = LEON ? 15 : 12;
-  assert(s->sound.init(MIN_VOLUME));
-
-  int draws = 0;
-
-  while (!do_exit) {
-    bool should_swap = false;
-    if (!s->started) {
-      // Delay a while to avoid 9% cpu usage while car is not started and user is keeping touching on the screen.
-      // Don't hold the lock while sleeping, so that vision_connect_thread have chances to get the lock.
-      usleep(30 * 1000);
-    }
-    pthread_mutex_lock(&s->lock);
-    double u1 = millis_since_boot();
-
-    // light sensor is only exposed on EONs
-    float clipped_brightness = (s->light_sensor*brightness_m) + brightness_b;
-    if (clipped_brightness > 512) clipped_brightness = 512;
-    smooth_brightness = clipped_brightness * 0.01 + smooth_brightness * 0.99;
-    if (smooth_brightness > 255) smooth_brightness = 255;
-    ui_set_brightness(s, (int)smooth_brightness);
-
-    // resize vision for collapsing sidebar
-    const bool hasSidebar = !s->scene.uilayout_sidebarcollapsed;
-    s->scene.ui_viz_rx = hasSidebar ? box_x : (box_x - sbr_w + (bdr_s * 2));
-    s->scene.ui_viz_rw = hasSidebar ? box_w : (box_w + sbr_w - (bdr_s * 2));
-    s->scene.ui_viz_ro = hasSidebar ? -(sbr_w - 6 * bdr_s) : 0;
-
-    // poll for touch events
-    int touch_x = -1, touch_y = -1;
-    int touched = touch_poll(&touch, &touch_x, &touch_y, 0);
-    if (touched == 1) {
-      set_awake(s, true);
-      handle_sidebar_touch(s, touch_x, touch_y);
-      handle_vision_touch(s, touch_x, touch_y);
-    }
-
-    if (!s->started) {
-      // always process events offroad
-      check_messages(s);
-
-      if (s->started) {
-        s->controls_timeout = 5 * UI_FREQ;
-      }
-    } else {
-      set_awake(s, true);
-      // Car started, fetch a new rgb image from ipc
-      if (s->vision_connected){
-        ui_update(s);
-      }
-
-      check_messages(s);
-
-      // Visiond process is just stopped, force a redraw to make screen blank again.
-      if (!s->started) {
-        s->scene.uilayout_sidebarcollapsed = false;
-        ui_draw(s);
-        glFinish();
-        should_swap = true;
-      }
-    }
-
-    // manage wakefulness
-    if (s->awake_timeout > 0) {
-      s->awake_timeout--;
-    } else {
-      set_awake(s, false);
-    }
-
-    // manage hardware disconnect
-    if (s->hardware_timeout > 0) {
-      s->hardware_timeout--;
-    } else {
-      s->scene.hwType = cereal::HealthData::HwType::UNKNOWN;
-    }
-
-    // Don't waste resources on drawing in case screen is off
-    if (s->awake) {
-      ui_draw(s);
-      glFinish();
-      should_swap = true;
-    }
-
-    s->sound.setVolume(fmin(MAX_VOLUME, MIN_VOLUME + s->scene.controls_state.getVEgo() / 5)); // up one notch every 5 m/s
-
-    if (s->controls_timeout > 0) {
-      s->controls_timeout--;
-    } else if (s->started && !s->scene.frontview) {
-      if (!s->controls_seen) {
-        // car is started, but controlsState hasn't been seen at all
-        s->scene.alert_text1 = "openpilot Unavailable";
-        s->scene.alert_text2 = "Waiting for controls to start";
-        s->scene.alert_size = cereal::ControlsState::AlertSize::MID;
-      } else {
-        // car is started, but controls is lagging or died
-        LOGE("Controls unresponsive");
-
-        if (s->scene.alert_text2 != "Controls Unresponsive") {
-          s->sound.play(AudibleAlert::CHIME_WARNING_REPEAT);
-        }
-
-        s->scene.alert_text1 = "TAKE CONTROL IMMEDIATELY";
-        s->scene.alert_text2 = "Controls Unresponsive";
-        s->scene.alert_size = cereal::ControlsState::AlertSize::FULL;
-        update_status(s, STATUS_ALERT);
-      }
-      ui_draw_vision_alert(s, s->scene.alert_size, s->status, s->scene.alert_text1.c_str(), s->scene.alert_text2.c_str());
-    }
-
-    read_param_timeout(&s->is_metric, "IsMetric", &s->is_metric_timeout);
-    read_param_timeout(&s->longitudinal_control, "LongitudinalControl", &s->longitudinal_control_timeout);
-    read_param_timeout(&s->limit_set_speed, "LimitSetSpeed", &s->limit_set_speed_timeout);
-    read_param_timeout(&s->speed_lim_off, "SpeedLimitOffset", &s->limit_set_speed_timeout);
-    int param_read = read_param_timeout(&s->last_athena_ping, "LastAthenaPingTime", &s->last_athena_ping_timeout);
-    if (param_read != -1) { // Param was updated this loop
-      if (param_read != 0) { // Failed to read param
-        s->scene.athenaStatus = NET_DISCONNECTED;
-      } else if (nanos_since_boot() - s->last_athena_ping < 70e9) {
-        s->scene.athenaStatus = NET_CONNECTED;
-      } else {
-        s->scene.athenaStatus = NET_ERROR;
-      }
-    }
-    update_offroad_layout_state(s);
-
-    pthread_mutex_unlock(&s->lock);
-
-    // the bg thread needs to be scheduled, so the main thread needs time without the lock
-    // safe to do this outside the lock?
-    if (should_swap) {
-      double u2 = millis_since_boot();
-      if (u2-u1 > 66) {
-        // warn on sub 15fps
-        LOGW("slow frame(%d) time: %.2f", draws, u2-u1);
-      }
-      draws++;
-      framebuffer_swap(s->fb);
-    }
-  }
-
-  set_awake(s, true);
-
-  // wake up bg thread to exit
-  pthread_mutex_lock(&s->lock);
-  pthread_mutex_unlock(&s->lock);
-
-#ifdef QCOM
-  // join light_sensor_thread?
-#endif
-
-  err = pthread_join(connect_thread_handle, NULL);
-  assert(err == 0);
-  delete s->sm;
-  delete s->pm;
-  return 0;
-}

selfdrive/ui/ui.hpp

@@ -12,7 +12,11 @@
 #define nvgCreate nvgCreateGLES3
 #endif
 #include <atomic>
+
+#include <string>
+#include <sstream>
 #include <pthread.h>
+
 #include "nanovg.h"
 
 #include "common/mat.h"
@@ -20,6 +24,7 @@
 #include "common/visionimg.h"
 #include "common/framebuffer.h"
 #include "common/modeldata.h"
+#include "common/params.h"
 #include "sound.hpp"
 
 #define STATUS_STOPPED 0
@@ -48,12 +53,19 @@
 //#define SHOW_SPEEDLIMIT 1
 //#define DEBUG_TURN
 
+// TODO: Detect dynamically
+#ifdef QCOM2
+const int vwp_w = 2160;
+#else
 const int vwp_w = 1920;
+#endif
+
 const int vwp_h = 1080;
 const int nav_w = 640;
 const int nav_ww= 760;
 const int sbr_w = 300;
 const int bdr_s = 30;
+
 const int box_x = sbr_w+bdr_s;
 const int box_y = bdr_s;
 const int box_w = vwp_w-sbr_w-(bdr_s*2);
@@ -243,12 +255,49 @@ typedef struct UIState {
   Sound sound;
 } UIState;
 
-// API
+
-void ui_draw_vision_alert(UIState *s, cereal::ControlsState::AlertSize va_size, int va_color,
+void ui_init(UIState *s);
-                          const char* va_text1, const char* va_text2);
+void ui_update(UIState *s);
-void ui_draw(UIState *s);
+void ui_update_sizes(UIState *s);
-void ui_draw_sidebar(UIState *s);
+
-void ui_draw_image(NVGcontext *vg, float x, float y, float w, float h, int image, float alpha);
+void* vision_connect_thread(void *args);
-void ui_draw_rect(NVGcontext *vg, float x, float y, float w, float h, NVGcolor color, float r = 0, int width = 0);
+void check_messages(UIState *s);
-void ui_draw_rect(NVGcontext *vg, float x, float y, float w, float h, NVGpaint &paint, float r = 0);
+void update_status(UIState *s, int status);
-void ui_nvg_init(UIState *s);
+
+
+int write_param_float(float param, const char* param_name, bool persistent_param = false);
+template <class T>
+int read_param(T* param, const char *param_name, bool persistent_param = false){
+  T param_orig = *param;
+  char *value;
+  size_t sz;
+
+  int result = read_db_value(param_name, &value, &sz, persistent_param);
+  if (result == 0){
+    std::string s = std::string(value, sz); // value is not null terminated
+    free(value);
+
+    // Parse result
+    std::istringstream iss(s);
+    iss >> *param;
+
+    // Restore original value if parsing failed
+    if (iss.fail()) {
+      *param = param_orig;
+      result = -1;
+    }
+  }
+  return result;
+}
+
+template <class T>
+int read_param_timeout(T* param, const char* param_name, int* timeout, bool persistent_param = false) {
+  int result = -1;
+  if (*timeout > 0){
+    (*timeout)--;
+  } else {
+    *timeout = 2 * UI_FREQ; // 0.5Hz
+    result = read_param(param, param_name, persistent_param);
+  }
+  return result;
+}

tools/mac_setup.sh

@@ -20,6 +20,7 @@ brew install capnp \
              libtool \
              llvm \
              pyenv \
+             qt5 \
              zeromq
 
 # Detect shell and pick correct RC file.

tools/ubuntu_setup.sh

@@ -41,6 +41,7 @@ sudo apt-get update && sudo apt-get install -y \
     opencl-headers \
     python-dev \
     python-pip \
+    qt5-default \
     screen \
     sudo \
     vim \