diff --git a/selfdrive/ui/SConscript b/selfdrive/ui/SConscript index da16c12d14..52e4e0c42c 100644 --- a/selfdrive/ui/SConscript +++ b/selfdrive/ui/SConscript @@ -56,10 +56,9 @@ qt_env.Program("qt/text", ["qt/text.cc"], LIBS=qt_libs) qt_env.Program("qt/spinner", ["qt/spinner.cc"], LIBS=qt_libs) # build main UI -qt_src = ["main.cc", "ui.cc", "paint.cc", "qt/sidebar.cc", "qt/onroad.cc", +qt_src = ["main.cc", "ui.cc", "qt/sidebar.cc", "qt/onroad.cc", "qt/window.cc", "qt/home.cc", "qt/offroad/settings.cc", - "qt/offroad/onboarding.cc", "qt/offroad/driverview.cc", - "#third_party/nanovg/nanovg.c"] + "qt/offroad/onboarding.cc", "qt/offroad/driverview.cc"] qt_env.Program("_ui", qt_src + [asset_obj], LIBS=qt_libs) diff --git a/selfdrive/ui/paint.cc b/selfdrive/ui/paint.cc deleted file mode 100644 index a8023801ca..0000000000 --- a/selfdrive/ui/paint.cc +++ /dev/null @@ -1,175 +0,0 @@ -#include "selfdrive/ui/paint.h" - -#include - -#ifdef __APPLE__ -#include -#define NANOVG_GL3_IMPLEMENTATION -#define nvgCreate nvgCreateGL3 -#else -#include -#define NANOVG_GLES3_IMPLEMENTATION -#define nvgCreate nvgCreateGLES3 -#endif - -#define NANOVG_GLES3_IMPLEMENTATION -#include -#include - -#include "selfdrive/hardware/hw.h" - -static void draw_chevron(UIState *s, float x, float y, float sz, NVGcolor fillColor, NVGcolor glowColor) { - // glow - float g_xo = sz/5; - float g_yo = sz/10; - nvgBeginPath(s->vg); - nvgMoveTo(s->vg, x+(sz*1.35)+g_xo, y+sz+g_yo); - nvgLineTo(s->vg, x, y-g_xo); - nvgLineTo(s->vg, x-(sz*1.35)-g_xo, y+sz+g_yo); - nvgClosePath(s->vg); - nvgFillColor(s->vg, glowColor); - nvgFill(s->vg); - - // chevron - nvgBeginPath(s->vg); - nvgMoveTo(s->vg, x+(sz*1.25), y+sz); - nvgLineTo(s->vg, x, y); - nvgLineTo(s->vg, x-(sz*1.25), y+sz); - nvgClosePath(s->vg); - nvgFillColor(s->vg, fillColor); - nvgFill(s->vg); -} - -static void draw_lead(UIState *s, const cereal::RadarState::LeadData::Reader &lead_data, const vertex_data &vd) { - // Draw lead car indicator - auto [x, y] = vd; - - float fillAlpha = 0; - float speedBuff = 10.; - float leadBuff = 40.; - float d_rel = lead_data.getDRel(); - float v_rel = lead_data.getVRel(); - if (d_rel < leadBuff) { - fillAlpha = 255*(1.0-(d_rel/leadBuff)); - if (v_rel < 0) { - fillAlpha += 255*(-1*(v_rel/speedBuff)); - } - fillAlpha = (int)(fmin(fillAlpha, 255)); - } - - float sz = std::clamp((25 * 30) / (d_rel / 3 + 30), 15.0f, 30.0f) * 2.35; - x = std::clamp(x, 0.f, s->fb_w - sz / 2); - y = std::fmin(s->fb_h - sz * .6, y); - draw_chevron(s, x, y, sz, nvgRGBA(201, 34, 49, fillAlpha), COLOR_YELLOW); -} - -static void ui_draw_line(UIState *s, const line_vertices_data &vd, NVGcolor *color, NVGpaint *paint) { - if (vd.cnt == 0) return; - - const vertex_data *v = &vd.v[0]; - nvgBeginPath(s->vg); - nvgMoveTo(s->vg, v[0].x, v[0].y); - for (int i = 1; i < vd.cnt; i++) { - nvgLineTo(s->vg, v[i].x, v[i].y); - } - nvgClosePath(s->vg); - if (color) { - nvgFillColor(s->vg, *color); - } else if (paint) { - nvgFillPaint(s->vg, *paint); - } - nvgFill(s->vg); -} - -static void ui_draw_vision_lane_lines(UIState *s) { - const UIScene &scene = s->scene; - NVGpaint track_bg; - if (!scene.end_to_end) { - // paint lanelines - for (int i = 0; i < std::size(scene.lane_line_vertices); i++) { - NVGcolor color = nvgRGBAf(1.0, 1.0, 1.0, scene.lane_line_probs[i]); - ui_draw_line(s, scene.lane_line_vertices[i], &color, nullptr); - } - - // paint road edges - for (int i = 0; i < std::size(scene.road_edge_vertices); i++) { - NVGcolor color = nvgRGBAf(1.0, 0.0, 0.0, std::clamp(1.0 - scene.road_edge_stds[i], 0.0, 1.0)); - ui_draw_line(s, scene.road_edge_vertices[i], &color, nullptr); - } - track_bg = nvgLinearGradient(s->vg, s->fb_w, s->fb_h, s->fb_w, s->fb_h * .4, - COLOR_WHITE, COLOR_WHITE_ALPHA(0)); - } else { - track_bg = nvgLinearGradient(s->vg, s->fb_w, s->fb_h, s->fb_w, s->fb_h * .4, - COLOR_RED, COLOR_RED_ALPHA(0)); - } - // paint path - ui_draw_line(s, scene.track_vertices, nullptr, &track_bg); -} - -// Draw all world space objects. -static void ui_draw_world(UIState *s) { - nvgScissor(s->vg, 0, 0, s->fb_w, s->fb_h); - - // Draw lane edges and vision/mpc tracks - ui_draw_vision_lane_lines(s); - - // Draw lead indicators if openpilot is handling longitudinal - if (s->scene.longitudinal_control) { - auto lead_one = (*s->sm)["radarState"].getRadarState().getLeadOne(); - auto lead_two = (*s->sm)["radarState"].getRadarState().getLeadTwo(); - if (lead_one.getStatus()) { - draw_lead(s, lead_one, s->scene.lead_vertices[0]); - } - if (lead_two.getStatus() && (std::abs(lead_one.getDRel() - lead_two.getDRel()) > 3.0)) { - draw_lead(s, lead_two, s->scene.lead_vertices[1]); - } - } - nvgResetScissor(s->vg); -} - -static void ui_draw_vision(UIState *s) { - const UIScene *scene = &s->scene; - // Draw augmented elements - if (scene->world_objects_visible) { - ui_draw_world(s); - } -} - -void ui_draw(UIState *s, int w, int h) { - // Update intrinsics matrix after possible wide camera toggle change - if (s->fb_w != w || s->fb_h != h) { - ui_resize(s, w, h); - } - glEnable(GL_BLEND); - glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); - nvgBeginFrame(s->vg, s->fb_w, s->fb_h, 1.0f); - ui_draw_vision(s); - nvgEndFrame(s->vg); - glDisable(GL_BLEND); -} - -void ui_nvg_init(UIState *s) { - // on EON, we enable MSAA - s->vg = Hardware::EON() ? nvgCreate(0) : nvgCreate(NVG_ANTIALIAS | NVG_STENCIL_STROKES | NVG_DEBUG); - assert(s->vg); -} - -void ui_resize(UIState *s, int width, int height) { - s->fb_w = width; - s->fb_h = height; - - auto intrinsic_matrix = s->wide_camera ? ecam_intrinsic_matrix : fcam_intrinsic_matrix; - float zoom = ZOOM / intrinsic_matrix.v[0]; - if (s->wide_camera) { - zoom *= 0.5; - } - - // Apply transformation such that video pixel coordinates match video - // 1) Put (0, 0) in the middle of the video - // 2) Apply same scaling as video - // 3) Put (0, 0) in top left corner of video - s->car_space_transform.reset(); - s->car_space_transform.translate(width / 2, height / 2 + y_offset) - .scale(zoom, zoom) - .translate(-intrinsic_matrix.v[2], -intrinsic_matrix.v[5]); -} diff --git a/selfdrive/ui/paint.h b/selfdrive/ui/paint.h deleted file mode 100644 index c888e3cb65..0000000000 --- a/selfdrive/ui/paint.h +++ /dev/null @@ -1,8 +0,0 @@ -#pragma once - -#include "selfdrive/ui/ui.h" - -void ui_draw(UIState *s, int w, int h); -void ui_nvg_init(UIState *s); -void ui_resize(UIState *s, int width, int height); -void ui_update_params(UIState *s); diff --git a/selfdrive/ui/qt/onroad.cc b/selfdrive/ui/qt/onroad.cc index 8902e436c9..af4045e1ec 100644 --- a/selfdrive/ui/qt/onroad.cc +++ b/selfdrive/ui/qt/onroad.cc @@ -3,7 +3,6 @@ #include #include "selfdrive/common/timing.h" -#include "selfdrive/ui/paint.h" #include "selfdrive/ui/qt/util.h" #ifdef ENABLE_MAPS #include "selfdrive/ui/qt/maps/map.h" @@ -266,14 +265,105 @@ void NvgWindow::initializeGL() { qInfo() << "OpenGL renderer:" << QString((const char*)glGetString(GL_RENDERER)); qInfo() << "OpenGL language version:" << QString((const char*)glGetString(GL_SHADING_LANGUAGE_VERSION)); - ui_nvg_init(&QUIState::ui_state); prev_draw_t = millis_since_boot(); setBackgroundColor(bg_colors[STATUS_DISENGAGED]); } +void NvgWindow::updateFrameMat(int w, int h) { + CameraViewWidget::updateFrameMat(w, h); + + UIState *s = &QUIState::ui_state; + s->fb_w = w; + s->fb_h = h; + auto intrinsic_matrix = s->wide_camera ? ecam_intrinsic_matrix : fcam_intrinsic_matrix; + float zoom = ZOOM / intrinsic_matrix.v[0]; + if (s->wide_camera) { + zoom *= 0.5; + } + // Apply transformation such that video pixel coordinates match video + // 1) Put (0, 0) in the middle of the video + // 2) Apply same scaling as video + // 3) Put (0, 0) in top left corner of video + s->car_space_transform.reset(); + s->car_space_transform.translate(w / 2, h / 2 + y_offset) + .scale(zoom, zoom) + .translate(-intrinsic_matrix.v[2], -intrinsic_matrix.v[5]); +} + +void NvgWindow::drawLaneLines(QPainter &painter, const UIScene &scene) { + if (!scene.end_to_end) { + // lanelines + for (int i = 0; i < std::size(scene.lane_line_vertices); ++i) { + painter.setBrush(QColor::fromRgbF(1.0, 1.0, 1.0, scene.lane_line_probs[i])); + painter.drawPolygon(scene.lane_line_vertices[i].v, scene.lane_line_vertices[i].cnt); + } + // road edges + for (int i = 0; i < std::size(scene.road_edge_vertices); ++i) { + painter.setBrush(QColor::fromRgbF(1.0, 0, 0, std::clamp(1.0 - scene.road_edge_stds[i], 0.0, 1.0))); + painter.drawPolygon(scene.road_edge_vertices[i].v, scene.road_edge_vertices[i].cnt); + } + } + // paint path + QLinearGradient bg(0, height(), 0, height() / 4); + bg.setColorAt(0, scene.end_to_end ? redColor() : QColor(255, 255, 255)); + bg.setColorAt(1, scene.end_to_end ? redColor(0) : QColor(255, 255, 255, 0)); + painter.setBrush(bg); + painter.drawPolygon(scene.track_vertices.v, scene.track_vertices.cnt); +} + +void NvgWindow::drawLead(QPainter &painter, const cereal::ModelDataV2::LeadDataV3::Reader &lead_data, const QPointF &vd) { + const float speedBuff = 10.; + const float leadBuff = 40.; + const float d_rel = lead_data.getX()[0]; + const float v_rel = lead_data.getV()[0]; + + float fillAlpha = 0; + if (d_rel < leadBuff) { + fillAlpha = 255 * (1.0 - (d_rel / leadBuff)); + if (v_rel < 0) { + fillAlpha += 255 * (-1 * (v_rel / speedBuff)); + } + fillAlpha = (int)(fmin(fillAlpha, 255)); + } + + float sz = std::clamp((25 * 30) / (d_rel / 3 + 30), 15.0f, 30.0f) * 2.35; + float x = std::clamp((float)vd.x(), 0.f, width() - sz / 2); + float y = std::fmin(height() - sz * .6, (float)vd.y()); + + float g_xo = sz / 5; + float g_yo = sz / 10; + + QPointF glow[] = {{x + (sz * 1.35) + g_xo, y + sz + g_yo}, {x, y - g_xo}, {x - (sz * 1.35) - g_xo, y + sz + g_yo}}; + painter.setBrush(QColor(218, 202, 37, 255)); + painter.drawPolygon(glow, std::size(glow)); + + // chevron + QPointF chevron[] = {{x + (sz * 1.25), y + sz}, {x, y}, {x - (sz * 1.25), y + sz}}; + painter.setBrush(redColor(fillAlpha)); + painter.drawPolygon(chevron, std::size(chevron)); +} + void NvgWindow::paintGL() { CameraViewWidget::paintGL(); - ui_draw(&QUIState::ui_state, width(), height()); + + UIState *s = &QUIState::ui_state; + if (s->scene.world_objects_visible) { + QPainter painter(this); + painter.setRenderHint(QPainter::Antialiasing); + painter.setPen(Qt::NoPen); + + drawLaneLines(painter, s->scene); + + if (s->scene.longitudinal_control) { + auto leads = (*s->sm)["modelV2"].getModelV2().getLeadsV3(); + if (leads[0].getProb() > .5) { + drawLead(painter, leads[0], s->scene.lead_vertices[0]); + } + if (leads[1].getProb() > .5 && (std::abs(leads[1].getX()[0] - leads[0].getX()[0]) > 3.0)) { + drawLead(painter, leads[1], s->scene.lead_vertices[1]); + } + } + } double cur_draw_t = millis_since_boot(); double dt = cur_draw_t - prev_draw_t; @@ -286,6 +376,7 @@ void NvgWindow::paintGL() { void NvgWindow::showEvent(QShowEvent *event) { CameraViewWidget::showEvent(event); + ui_update_params(&QUIState::ui_state); prev_draw_t = millis_since_boot(); } diff --git a/selfdrive/ui/qt/onroad.h b/selfdrive/ui/qt/onroad.h index 315aff54f0..37460c8ba6 100644 --- a/selfdrive/ui/qt/onroad.h +++ b/selfdrive/ui/qt/onroad.h @@ -72,6 +72,10 @@ protected: void paintGL() override; void initializeGL() override; void showEvent(QShowEvent *event) override; + void updateFrameMat(int w, int h) override; + void drawLaneLines(QPainter &painter, const UIScene &scene); + void drawLead(QPainter &painter, const cereal::ModelDataV2::LeadDataV3::Reader &lead_data, const QPointF &vd); + inline QColor redColor(int alpha = 255) { return QColor(201, 34, 49, alpha); } double prev_draw_t = 0; }; diff --git a/selfdrive/ui/qt/util.cc b/selfdrive/ui/qt/util.cc index 1661c6c5d0..b1defb0081 100644 --- a/selfdrive/ui/qt/util.cc +++ b/selfdrive/ui/qt/util.cc @@ -85,6 +85,7 @@ void setQtSurfaceFormat() { #else fmt.setRenderableType(QSurfaceFormat::OpenGLES); #endif + fmt.setSamples(16); QSurfaceFormat::setDefaultFormat(fmt); } diff --git a/selfdrive/ui/qt/widgets/cameraview.cc b/selfdrive/ui/qt/widgets/cameraview.cc index f09b1b1796..4203d146e0 100644 --- a/selfdrive/ui/qt/widgets/cameraview.cc +++ b/selfdrive/ui/qt/widgets/cameraview.cc @@ -1,12 +1,18 @@ #include "selfdrive/ui/qt/widgets/cameraview.h" +#ifdef __APPLE__ +#include +#else +#include +#endif + #include #include namespace { const char frame_vertex_shader[] = -#ifdef NANOVG_GL3_IMPLEMENTATION +#ifdef __APPLE__ "#version 150 core\n" #else "#version 300 es\n" @@ -21,7 +27,7 @@ const char frame_vertex_shader[] = "}\n"; const char frame_fragment_shader[] = -#ifdef NANOVG_GL3_IMPLEMENTATION +#ifdef __APPLE__ "#version 150 core\n" #else "#version 300 es\n" diff --git a/selfdrive/ui/qt/widgets/cameraview.h b/selfdrive/ui/qt/widgets/cameraview.h index 146528cd23..4cfba3c6fb 100644 --- a/selfdrive/ui/qt/widgets/cameraview.h +++ b/selfdrive/ui/qt/widgets/cameraview.h @@ -7,6 +7,7 @@ #include #include #include "cereal/visionipc/visionipc_client.h" +#include "selfdrive/camerad/cameras/camera_common.h" #include "selfdrive/common/visionimg.h" #include "selfdrive/ui/ui.h" @@ -32,7 +33,7 @@ protected: void showEvent(QShowEvent *event) override; void hideEvent(QHideEvent *event) override; void mouseReleaseEvent(QMouseEvent *event) override { emit clicked(); } - void updateFrameMat(int w, int h); + virtual void updateFrameMat(int w, int h); void vipcThread(); bool zoomed_view; diff --git a/selfdrive/ui/soundd/sound.cc b/selfdrive/ui/soundd/sound.cc index 766c37a51c..4e88f41b1c 100644 --- a/selfdrive/ui/soundd/sound.cc +++ b/selfdrive/ui/soundd/sound.cc @@ -1,5 +1,7 @@ #include "selfdrive/ui/soundd/sound.h" +#include + #include #include #include diff --git a/selfdrive/ui/ui.cc b/selfdrive/ui/ui.cc index f9a473bc83..53d73c1043 100644 --- a/selfdrive/ui/ui.cc +++ b/selfdrive/ui/ui.cc @@ -1,11 +1,11 @@ #include "selfdrive/ui/ui.h" -#include - #include #include -#include +#include "common/transformations/orientation.hpp" +#include "selfdrive/common/params.h" +#include "selfdrive/common/swaglog.h" #include "selfdrive/common/util.h" #include "selfdrive/common/watchdog.h" #include "selfdrive/hardware/hw.h" @@ -14,10 +14,9 @@ #define BACKLIGHT_TS 10.00 #define BACKLIGHT_OFFROAD 50 - // Projects a point in car to space to the corresponding point in full frame // image space. -static bool calib_frame_to_full_frame(const UIState *s, float in_x, float in_y, float in_z, vertex_data *out) { +static bool calib_frame_to_full_frame(const UIState *s, float in_x, float in_y, float in_z, QPointF *out) { const float margin = 500.0f; const QRectF clip_region{-margin, -margin, s->fb_w + 2 * margin, s->fb_h + 2 * margin}; @@ -28,8 +27,7 @@ static bool calib_frame_to_full_frame(const UIState *s, float in_x, float in_y, // Project. QPointF point = s->car_space_transform.map(QPointF{KEp.v[0] / KEp.v[2], KEp.v[1] / KEp.v[2]}); if (clip_region.contains(point)) { - out->x = point.x(); - out->y = point.y(); + *out = point; return true; } return false; @@ -57,7 +55,7 @@ static void update_leads(UIState *s, const cereal::RadarState::Reader &radar_sta static void update_line_data(const UIState *s, const cereal::ModelDataV2::XYZTData::Reader &line, float y_off, float z_off, line_vertices_data *pvd, int max_idx) { const auto line_x = line.getX(), line_y = line.getY(), line_z = line.getZ(); - vertex_data *v = &pvd->v[0]; + QPointF *v = &pvd->v[0]; for (int i = 0; i <= max_idx; i++) { v += calib_frame_to_full_frame(s, line_x[i], line_y[i] - y_off, line_z[i] + z_off, v); } @@ -110,10 +108,10 @@ static void update_state(UIState *s) { UIScene &scene = s->scene; s->running_time = 1e-9 * (nanos_since_boot() - sm["deviceState"].getDeviceState().getStartedMonoTime()); - if (sm.updated("modelV2") && s->vg) { + if (sm.updated("modelV2")) { update_model(s, sm["modelV2"].getModelV2()); } - if (sm.updated("radarState") && s->vg) { + if (sm.updated("radarState")) { std::optional line; if (sm.rcv_frame("modelV2") > 0) { line = sm["modelV2"].getModelV2().getPosition(); diff --git a/selfdrive/ui/ui.h b/selfdrive/ui/ui.h index 52ba882c2b..3b3d11b67a 100644 --- a/selfdrive/ui/ui.h +++ b/selfdrive/ui/ui.h @@ -1,29 +1,17 @@ #pragma once -#include #include #include -#include #include #include #include #include -#include "nanovg.h" #include "cereal/messaging/messaging.h" -#include "common/transformations/orientation.hpp" -#include "selfdrive/camerad/cameras/camera_common.h" -#include "selfdrive/common/mat.h" #include "selfdrive/common/modeldata.h" #include "selfdrive/common/params.h" -#include "selfdrive/common/util.h" - -#define COLOR_WHITE nvgRGBA(255, 255, 255, 255) -#define COLOR_WHITE_ALPHA(x) nvgRGBA(255, 255, 255, x) -#define COLOR_RED_ALPHA(x) nvgRGBA(201, 34, 49, x) -#define COLOR_YELLOW nvgRGBA(218, 202, 37, 255) -#define COLOR_RED nvgRGBA(201, 34, 49, 255) +#include "selfdrive/common/timing.h" const int bdr_s = 30; const int header_h = 420; @@ -87,16 +75,11 @@ const QColor bg_colors [] = { }; typedef struct { - float x, y; -} vertex_data; - -typedef struct { - vertex_data v[TRAJECTORY_SIZE * 2]; + QPointF v[TRAJECTORY_SIZE * 2]; int cnt; } line_vertices_data; typedef struct UIScene { - mat3 view_from_calib; bool world_objects_visible; @@ -110,7 +93,7 @@ typedef struct UIScene { line_vertices_data road_edge_vertices[2]; // lead - vertex_data lead_vertices[2]; + QPointF lead_vertices[2]; float light_sensor, accel_sensor, gyro_sensor; bool started, ignition, is_metric, longitudinal_control, end_to_end; @@ -119,7 +102,6 @@ typedef struct UIScene { typedef struct UIState { int fb_w = 0, fb_h = 0; - NVGcontext *vg; std::unique_ptr sm; @@ -189,3 +171,5 @@ public slots: void setAwake(bool on, bool reset); void update(const UIState &s); }; + +void ui_update_params(UIState *s);