Merge remote-tracking branch 'upstream/master' into long-man

pull/33527/head
Shane Smiskol 10 months ago
commit 8d414c249c
  1. 2
      .gitmodules
  2. 1
      cereal/car.capnp
  3. 1
      cereal/services.py
  4. 9
      selfdrive/selfdrived/events.py
  5. 4
      selfdrive/ui/SConscript
  6. 5
      selfdrive/ui/qt/home.cc
  7. 19
      selfdrive/ui/qt/network/networking.cc
  8. 3
      selfdrive/ui/qt/network/networking.h
  9. 7
      selfdrive/ui/qt/network/wifi_manager.cc
  10. 1
      selfdrive/ui/qt/network/wifi_manager.h
  11. 14
      selfdrive/ui/qt/offroad/driverview.cc
  12. 1
      selfdrive/ui/qt/offroad/driverview.h
  13. 14
      selfdrive/ui/qt/offroad/settings.cc
  14. 1
      selfdrive/ui/qt/offroad/settings.h
  15. 63
      selfdrive/ui/qt/onroad/annotated_camera.cc
  16. 2
      selfdrive/ui/qt/onroad/annotated_camera.h
  17. 2
      selfdrive/ui/qt/onroad/onroad_home.cc
  18. 48
      selfdrive/ui/qt/prime_state.cc
  19. 33
      selfdrive/ui/qt/prime_state.h
  20. 106
      selfdrive/ui/qt/widgets/cameraview.cc
  21. 16
      selfdrive/ui/qt/widgets/cameraview.h
  22. 4
      selfdrive/ui/qt/widgets/keyboard.cc
  23. 37
      selfdrive/ui/qt/widgets/prime.cc
  24. 3
      selfdrive/ui/qt/widgets/prime.h
  25. 1
      selfdrive/ui/tests/test_ui/run.py
  26. 8
      selfdrive/ui/translations/main_zh-CHS.ts
  27. 6
      selfdrive/ui/translations/main_zh-CHT.ts
  28. 80
      selfdrive/ui/ui.cc
  29. 61
      selfdrive/ui/ui.h
  30. 6
      selfdrive/ui/watch3.cc
  31. 2
      system/webrtc/tests/test_stream_session.py
  32. 2
      tinygrad_repo
  33. 6
      tools/cabana/videowidget.cc
  34. 2
      tools/cabana/videowidget.h

2
.gitmodules vendored

@ -15,4 +15,4 @@
url = ../../commaai/teleoprtc
[submodule "tinygrad"]
path = tinygrad_repo
url = https://github.com/tinygrad/tinygrad.git
url = https://github.com/commaai/tinygrad.git

@ -115,6 +115,7 @@ struct OnroadEvent @0x9b1657f34caf3ad3 {
actuatorsApiUnavailable @120;
espActive @121;
personalityChanged @122;
aeb @123;
radarCanErrorDEPRECATED @15;
communityFeatureDisallowedDEPRECATED @62;

@ -76,7 +76,6 @@ _services: dict[str, tuple] = {
# debug
"uiDebug": (True, 0., 1),
"alertDebug": (True, 0.),
"testJoystick": (True, 0.),
"roadEncodeData": (False, 20.),
"driverEncodeData": (False, 20.),
"wideRoadEncodeData": (False, 20.),

@ -388,6 +388,15 @@ EVENTS: dict[int, dict[str, Alert | AlertCallbackType]] = {
priority=Priority.LOWEST),
},
EventName.aeb: {
ET.PERMANENT: Alert(
"BRAKE!",
"Emergency Braking: Risk of Collision",
AlertStatus.critical, AlertSize.full,
Priority.HIGHEST, VisualAlert.fcw, AudibleAlert.none, 2.),
ET.NO_ENTRY: NoEntryAlert("AEB: Risk of Collision"),
},
EventName.stockAeb: {
ET.PERMANENT: Alert(
"BRAKE!",

@ -16,7 +16,7 @@ if arch == "Darwin":
qt_env['CXXFLAGS'] += ["-Wno-deprecated-declarations"]
qt_util = qt_env.Library("qt_util", ["#selfdrive/ui/qt/api.cc", "#selfdrive/ui/qt/util.cc"], LIBS=base_libs)
widgets_src = ["ui.cc", "qt/widgets/input.cc", "qt/widgets/wifi.cc",
widgets_src = ["qt/widgets/input.cc", "qt/widgets/wifi.cc", "qt/prime_state.cc",
"qt/widgets/ssh_keys.cc", "qt/widgets/toggle.cc", "qt/widgets/controls.cc",
"qt/widgets/offroad_alerts.cc", "qt/widgets/prime.cc", "qt/widgets/keyboard.cc",
"qt/widgets/scrollview.cc", "qt/widgets/cameraview.cc", "#third_party/qrcode/QrCode.cc",
@ -26,7 +26,7 @@ widgets = qt_env.Library("qt_widgets", widgets_src, LIBS=base_libs)
Export('widgets')
qt_libs = [widgets, qt_util] + base_libs
qt_src = ["main.cc", "qt/sidebar.cc", "qt/body.cc",
qt_src = ["main.cc", "ui.cc", "qt/sidebar.cc", "qt/body.cc",
"qt/window.cc", "qt/home.cc", "qt/offroad/settings.cc",
"qt/offroad/software_settings.cc", "qt/offroad/onboarding.cc",
"qt/offroad/driverview.cc", "qt/offroad/experimental_mode.cc",

@ -150,9 +150,8 @@ OffroadHome::OffroadHome(QWidget* parent) : QFrame(parent) {
left_widget->addWidget(new PrimeAdWidget);
left_widget->setStyleSheet("border-radius: 10px;");
left_widget->setCurrentIndex(uiState()->hasPrime() ? 0 : 1);
connect(uiState(), &UIState::primeChanged, [=](bool prime) {
left_widget->setCurrentIndex(prime ? 0 : 1);
connect(uiState()->prime_state, &PrimeState::changed, [left_widget]() {
left_widget->setCurrentIndex(uiState()->prime_state->isSubscribed() ? 0 : 1);
});
home_layout->addWidget(left_widget, 1);

@ -6,7 +6,6 @@
#include <QScrollBar>
#include <QStyle>
#include "selfdrive/ui/ui.h"
#include "selfdrive/ui/qt/qt_window.h"
#include "selfdrive/ui/qt/util.h"
#include "selfdrive/ui/qt/widgets/controls.h"
@ -73,6 +72,11 @@ Networking::Networking(QWidget* parent, bool show_advanced) : QFrame(parent) {
main_layout->setCurrentWidget(wifiScreen);
}
void Networking::setPrimeType(PrimeState::Type type) {
an->setGsmVisible(type == PrimeState::PRIME_TYPE_NONE || type == PrimeState::PRIME_TYPE_LITE);
wifi->ipv4_forward = (type == PrimeState::PRIME_TYPE_NONE || type == PrimeState::PRIME_TYPE_LITE);
}
void Networking::refresh() {
wifiWidget->refresh();
an->refresh();
@ -204,17 +208,16 @@ AdvancedNetworking::AdvancedNetworking(QWidget* parent, WifiManager* wifi): QWid
// Set initial config
wifi->updateGsmSettings(roamingEnabled, QString::fromStdString(params.get("GsmApn")), metered);
connect(uiState(), &UIState::primeTypeChanged, this, [=](PrimeType prime_type) {
bool gsmVisible = prime_type == PrimeType::PRIME_TYPE_NONE || prime_type == PrimeType::PRIME_TYPE_LITE;
roamingToggle->setVisible(gsmVisible);
editApnButton->setVisible(gsmVisible);
meteredToggle->setVisible(gsmVisible);
});
main_layout->addWidget(new ScrollView(list, this));
main_layout->addStretch(1);
}
void AdvancedNetworking::setGsmVisible(bool visible) {
roamingToggle->setVisible(visible);
editApnButton->setVisible(visible);
meteredToggle->setVisible(visible);
}
void AdvancedNetworking::refresh() {
ipLabel->setText(wifi->ipv4_address);
tetheringToggle->setEnabled(true);

@ -3,6 +3,7 @@
#include <vector>
#include "selfdrive/ui/qt/network/wifi_manager.h"
#include "selfdrive/ui/qt/prime_state.h"
#include "selfdrive/ui/qt/widgets/input.h"
#include "selfdrive/ui/qt/widgets/ssh_keys.h"
#include "selfdrive/ui/qt/widgets/toggle.h"
@ -56,6 +57,7 @@ class AdvancedNetworking : public QWidget {
Q_OBJECT
public:
explicit AdvancedNetworking(QWidget* parent = 0, WifiManager* wifi = 0);
void setGsmVisible(bool visible);
private:
LabelControl* ipLabel;
@ -81,6 +83,7 @@ class Networking : public QFrame {
public:
explicit Networking(QWidget* parent = 0, bool show_advanced = true);
void setPrimeType(PrimeState::Type type);
WifiManager* wifi = nullptr;
private:

@ -2,10 +2,6 @@
#include <utility>
#include "selfdrive/ui/ui.h"
#include "selfdrive/ui/qt/widgets/prime.h"
#include "common/params.h"
#include "common/swaglog.h"
#include "selfdrive/ui/qt/util.h"
@ -445,9 +441,6 @@ void WifiManager::addTetheringConnection() {
}
void WifiManager::tetheringActivated(QDBusPendingCallWatcher *call) {
int prime_type = uiState()->primeType();
int ipv4_forward = (prime_type == PrimeType::PRIME_TYPE_NONE || prime_type == PrimeType::PRIME_TYPE_LITE);
if (!ipv4_forward) {
QTimer::singleShot(5000, this, [=] {
qWarning() << "net.ipv4.ip_forward = 0";

@ -42,6 +42,7 @@ public:
QMap<QString, Network> seenNetworks;
QMap<QDBusObjectPath, QString> knownConnections;
QString ipv4_address;
bool ipv4_forward = false;
explicit WifiManager(QObject* parent);
void start();

@ -7,7 +7,7 @@
const int FACE_IMG_SIZE = 130;
DriverViewWindow::DriverViewWindow(QWidget* parent) : CameraWidget("camerad", VISION_STREAM_DRIVER, true, parent) {
DriverViewWindow::DriverViewWindow(QWidget* parent) : CameraWidget("camerad", VISION_STREAM_DRIVER, parent) {
face_img = loadPixmap("../assets/img_driver_face_static.png", {FACE_IMG_SIZE, FACE_IMG_SIZE});
QObject::connect(this, &CameraWidget::clicked, this, &DriverViewWindow::done);
QObject::connect(device(), &Device::interactiveTimeout, this, [this]() {
@ -75,3 +75,15 @@ void DriverViewWindow::paintGL() {
p.setOpacity(face_detected ? 1.0 : 0.2);
p.drawPixmap(img_x, img_y, face_img);
}
mat4 DriverViewWindow::calcFrameMatrix() {
const float driver_view_ratio = 2.0;
const float yscale = stream_height * driver_view_ratio / stream_width;
const float xscale = yscale * glHeight() / glWidth() * stream_width / stream_height;
return mat4{{
xscale, 0.0, 0.0, 0.0,
0.0, yscale, 0.0, 0.0,
0.0, 0.0, 1.0, 0.0,
0.0, 0.0, 0.0, 1.0,
}};
}

@ -12,6 +12,7 @@ signals:
void done();
protected:
mat4 calcFrameMatrix() override;
void showEvent(QShowEvent *event) override;
void hideEvent(QHideEvent *event) override;
void paintGL() override;

@ -247,8 +247,8 @@ DevicePanel::DevicePanel(SettingsWindow *parent) : ListWidget(parent) {
});
addItem(translateBtn);
QObject::connect(uiState(), &UIState::primeTypeChanged, [this] (PrimeType type) {
pair_device->setVisible(type == PrimeType::PRIME_TYPE_UNPAIRED);
QObject::connect(uiState()->prime_state, &PrimeState::changed, [this] (PrimeState::Type type) {
pair_device->setVisible(type == PrimeState::PRIME_TYPE_UNPAIRED);
});
QObject::connect(uiState(), &UIState::offroadTransition, [=](bool offroad) {
for (auto btn : findChildren<ButtonControl *>()) {
@ -335,11 +335,6 @@ void DevicePanel::poweroff() {
}
}
void DevicePanel::showEvent(QShowEvent *event) {
pair_device->setVisible(uiState()->primeType() == PrimeType::PRIME_TYPE_UNPAIRED);
ListWidget::showEvent(event);
}
void SettingsWindow::showEvent(QShowEvent *event) {
setCurrentPanel(0);
}
@ -386,9 +381,12 @@ SettingsWindow::SettingsWindow(QWidget *parent) : QFrame(parent) {
TogglesPanel *toggles = new TogglesPanel(this);
QObject::connect(this, &SettingsWindow::expandToggleDescription, toggles, &TogglesPanel::expandToggleDescription);
auto networking = new Networking(this);
QObject::connect(uiState()->prime_state, &PrimeState::changed, networking, &Networking::setPrimeType);
QList<QPair<QString, QWidget *>> panels = {
{tr("Device"), device},
{tr("Network"), new Networking(this)},
{tr("Network"), networking},
{tr("Toggles"), toggles},
{tr("Software"), new SoftwarePanel(this)},
};

@ -42,7 +42,6 @@ class DevicePanel : public ListWidget {
Q_OBJECT
public:
explicit DevicePanel(SettingsWindow *parent);
void showEvent(QShowEvent *event) override;
signals:
void reviewTrainingGuide();

@ -10,7 +10,8 @@
#include "selfdrive/ui/qt/util.h"
// Window that shows camera view and variety of info drawn on top
AnnotatedCameraWidget::AnnotatedCameraWidget(VisionStreamType type, QWidget* parent) : fps_filter(UI_FREQ, 3, 1. / UI_FREQ), CameraWidget("camerad", type, true, parent) {
AnnotatedCameraWidget::AnnotatedCameraWidget(VisionStreamType type, QWidget *parent)
: fps_filter(UI_FREQ, 3, 1. / UI_FREQ), CameraWidget("camerad", type, parent) {
pm = std::make_unique<PubMaster>(std::vector<const char*>{"uiDebug"});
main_layout = new QVBoxLayout(this);
@ -125,22 +126,54 @@ void AnnotatedCameraWidget::initializeGL() {
setBackgroundColor(bg_colors[STATUS_DISENGAGED]);
}
void AnnotatedCameraWidget::updateFrameMat() {
CameraWidget::updateFrameMat();
UIState *s = uiState();
mat4 AnnotatedCameraWidget::calcFrameMatrix() {
// Project point at "infinity" to compute x and y offsets
// to ensure this ends up in the middle of the screen
// for narrow come and a little lower for wide cam.
// TODO: use proper perspective transform?
// Select intrinsic matrix and calibration based on camera type
auto *s = uiState();
bool wide_cam = active_stream_type == VISION_STREAM_WIDE_ROAD;
const auto &intrinsic_matrix = wide_cam ? ECAM_INTRINSIC_MATRIX : FCAM_INTRINSIC_MATRIX;
const auto &calibration = wide_cam ? s->scene.view_from_wide_calib : s->scene.view_from_calib;
// Compute the calibration transformation matrix
const auto calib_transform = intrinsic_matrix * calibration;
float zoom = wide_cam ? 2.0 : 1.1;
Eigen::Vector3f inf(1000., 0., 0.);
auto Kep = calib_transform * inf;
int w = width(), h = height();
float center_x = intrinsic_matrix(0, 2);
float center_y = intrinsic_matrix(1, 2);
s->fb_w = w;
s->fb_h = h;
float max_x_offset = center_x * zoom - w / 2 - 5;
float max_y_offset = center_y * zoom - h / 2 - 5;
float x_offset = std::clamp<float>((Kep.x() / Kep.z() - center_x) * zoom, -max_x_offset, max_x_offset);
float y_offset = std::clamp<float>((Kep.y() / Kep.z() - center_y) * zoom, -max_y_offset, max_y_offset);
// Apply transformation such that video pixel coordinates match video
// 1) Put (0, 0) in the middle of the video
// 2) Apply same scaling as video
// 3) Put (0, 0) in top left corner of video
s->car_space_transform.reset();
s->car_space_transform.translate(w / 2 - x_offset, h / 2 - y_offset)
.scale(zoom, zoom)
.translate(-intrinsic_matrix.v[2], -intrinsic_matrix.v[5]);
Eigen::Matrix3f video_transform =(Eigen::Matrix3f() <<
zoom, 0.0f, (w / 2 - x_offset) - (center_x * zoom),
0.0f, zoom, (h / 2 - y_offset) - (center_y * zoom),
0.0f, 0.0f, 1.0f).finished();
s->car_space_transform = video_transform * calib_transform;
s->clip_region = rect().adjusted(-500, -500, 500, 500);
float zx = zoom * 2 * center_x / w;
float zy = zoom * 2 * center_y / h;
return mat4{{
zx, 0.0, 0.0, -x_offset / w * 2,
0.0, zy, 0.0, y_offset / h * 2,
0.0, 0.0, 1.0, 0.0,
0.0, 0.0, 0.0, 1.0,
}};
}
void AnnotatedCameraWidget::drawLaneLines(QPainter &painter, const UIState *s) {
@ -271,18 +304,8 @@ void AnnotatedCameraWidget::paintGL() {
wide_cam_requested = false;
}
wide_cam_requested = wide_cam_requested && sm["selfdriveState"].getSelfdriveState().getExperimentalMode();
// for replay of old routes, never go to widecam
wide_cam_requested = wide_cam_requested && s->scene.calibration_wide_valid;
}
CameraWidget::setStreamType(wide_cam_requested ? VISION_STREAM_WIDE_ROAD : VISION_STREAM_ROAD);
s->scene.wide_cam = CameraWidget::getStreamType() == VISION_STREAM_WIDE_ROAD;
if (s->scene.calibration_valid) {
auto calib = s->scene.wide_cam ? s->scene.view_from_wide_calib : s->scene.view_from_calib;
CameraWidget::updateCalibration(calib);
} else {
CameraWidget::updateCalibration(DEFAULT_CALIBRATION);
}
CameraWidget::setFrameId(model.getFrameId());
CameraWidget::paintGL();
}

@ -36,7 +36,7 @@ protected:
void paintGL() override;
void initializeGL() override;
void showEvent(QShowEvent *event) override;
void updateFrameMat() override;
mat4 calcFrameMatrix() override;
void drawLaneLines(QPainter &painter, const UIState *s);
void drawLead(QPainter &painter, const cereal::RadarState::LeadData::Reader &lead_data, const QPointF &vd);
void drawHud(QPainter &p);

@ -21,7 +21,7 @@ OnroadWindow::OnroadWindow(QWidget *parent) : QWidget(parent) {
split->addWidget(nvg);
if (getenv("DUAL_CAMERA_VIEW")) {
CameraWidget *arCam = new CameraWidget("camerad", VISION_STREAM_ROAD, true, this);
CameraWidget *arCam = new CameraWidget("camerad", VISION_STREAM_ROAD, this);
split->insertWidget(0, arCam);
}

@ -0,0 +1,48 @@
#include "selfdrive/ui/qt/prime_state.h"
#include <QJsonDocument>
#include "selfdrive/ui/qt/api.h"
#include "selfdrive/ui/qt/request_repeater.h"
#include "selfdrive/ui/qt/util.h"
PrimeState::PrimeState(QObject* parent) : QObject(parent) {
const char *env_prime_type = std::getenv("PRIME_TYPE");
auto type = env_prime_type ? env_prime_type : Params().get("PrimeType");
if (!type.empty()) {
prime_type = static_cast<PrimeState::Type>(std::atoi(type.c_str()));
}
if (auto dongleId = getDongleId()) {
QString url = CommaApi::BASE_URL + "/v1.1/devices/" + *dongleId + "/";
RequestRepeater* repeater = new RequestRepeater(this, url, "ApiCache_Device", 5);
QObject::connect(repeater, &RequestRepeater::requestDone, this, &PrimeState::handleReply);
}
// Emit the initial state change
QTimer::singleShot(1, [this]() { emit changed(prime_type); });
}
void PrimeState::handleReply(const QString& response, bool success) {
if (!success) return;
QJsonDocument doc = QJsonDocument::fromJson(response.toUtf8());
if (doc.isNull()) {
qDebug() << "JSON Parse failed on getting pairing and PrimeState status";
return;
}
QJsonObject json = doc.object();
bool is_paired = json["is_paired"].toBool();
auto type = static_cast<PrimeState::Type>(json["prime_type"].toInt());
setType(is_paired ? type : PrimeState::PRIME_TYPE_UNPAIRED);
}
void PrimeState::setType(PrimeState::Type type) {
if (type != prime_type) {
prime_type = type;
Params().put("PrimeType", std::to_string(prime_type));
emit changed(prime_type);
}
}

@ -0,0 +1,33 @@
#pragma once
#include <QObject>
class PrimeState : public QObject {
Q_OBJECT
public:
enum Type {
PRIME_TYPE_UNKNOWN = -2,
PRIME_TYPE_UNPAIRED = -1,
PRIME_TYPE_NONE = 0,
PRIME_TYPE_MAGENTA = 1,
PRIME_TYPE_LITE = 2,
PRIME_TYPE_BLUE = 3,
PRIME_TYPE_MAGENTA_NEW = 4,
PRIME_TYPE_PURPLE = 5,
};
PrimeState(QObject *parent);
void setType(PrimeState::Type type);
inline PrimeState::Type currentType() const { return prime_type; }
inline bool isSubscribed() const { return prime_type > PrimeState::PRIME_TYPE_NONE; }
signals:
void changed(PrimeState::Type prime_type);
private:
void handleReply(const QString &response, bool success);
PrimeState::Type prime_type = PrimeState::PRIME_TYPE_UNKNOWN;
};

@ -7,13 +7,7 @@
#endif
#include <cmath>
#include <set>
#include <string>
#include <utility>
#include <QApplication>
#include <QOpenGLBuffer>
#include <QOffscreenSurface>
namespace {
@ -66,40 +60,10 @@ const char frame_fragment_shader[] =
"}\n";
#endif
mat4 get_driver_view_transform(int screen_width, int screen_height, int stream_width, int stream_height) {
const float driver_view_ratio = 2.0;
const float yscale = stream_height * driver_view_ratio / stream_width;
const float xscale = yscale*screen_height/screen_width*stream_width/stream_height;
mat4 transform = (mat4){{
xscale, 0.0, 0.0, 0.0,
0.0, yscale, 0.0, 0.0,
0.0, 0.0, 1.0, 0.0,
0.0, 0.0, 0.0, 1.0,
}};
return transform;
}
mat4 get_fit_view_transform(float widget_aspect_ratio, float frame_aspect_ratio) {
float zx = 1, zy = 1;
if (frame_aspect_ratio > widget_aspect_ratio) {
zy = widget_aspect_ratio / frame_aspect_ratio;
} else {
zx = frame_aspect_ratio / widget_aspect_ratio;
}
const mat4 frame_transform = {{
zx, 0.0, 0.0, 0.0,
0.0, zy, 0.0, 0.0,
0.0, 0.0, 1.0, 0.0,
0.0, 0.0, 0.0, 1.0,
}};
return frame_transform;
}
} // namespace
CameraWidget::CameraWidget(std::string stream_name, VisionStreamType type, bool zoom, QWidget* parent) :
stream_name(stream_name), active_stream_type(type), requested_stream_type(type), zoomed_view(zoom), QOpenGLWidget(parent) {
CameraWidget::CameraWidget(std::string stream_name, VisionStreamType type, QWidget* parent) :
stream_name(stream_name), active_stream_type(type), requested_stream_type(type), QOpenGLWidget(parent) {
setAttribute(Qt::WA_OpaquePaintEvent);
qRegisterMetaType<std::set<VisionStreamType>>("availableStreams");
QObject::connect(this, &CameraWidget::vipcThreadConnected, this, &CameraWidget::vipcConnected, Qt::BlockingQueuedConnection);
@ -214,59 +178,19 @@ void CameraWidget::availableStreamsUpdated(std::set<VisionStreamType> streams) {
available_streams = streams;
}
void CameraWidget::updateFrameMat() {
int w = glWidth(), h = glHeight();
mat4 CameraWidget::calcFrameMatrix() {
// Scale the frame to fit the widget while maintaining the aspect ratio.
float widget_aspect_ratio = (float)width() / height();
float frame_aspect_ratio = (float)stream_width / stream_height;
float zx = std::min(frame_aspect_ratio / widget_aspect_ratio, 1.0f);
float zy = std::min(widget_aspect_ratio / frame_aspect_ratio, 1.0f);
if (zoomed_view) {
if (active_stream_type == VISION_STREAM_DRIVER) {
if (stream_width > 0 && stream_height > 0) {
frame_mat = get_driver_view_transform(w, h, stream_width, stream_height);
}
} else {
// Project point at "infinity" to compute x and y offsets
// to ensure this ends up in the middle of the screen
// for narrow come and a little lower for wide cam.
// TODO: use proper perspective transform?
if (active_stream_type == VISION_STREAM_WIDE_ROAD) {
intrinsic_matrix = ECAM_INTRINSIC_MATRIX;
zoom = 2.0;
} else {
intrinsic_matrix = FCAM_INTRINSIC_MATRIX;
zoom = 1.1;
}
const vec3 inf = {{1000., 0., 0.}};
const vec3 Ep = matvecmul3(calibration, inf);
const vec3 Kep = matvecmul3(intrinsic_matrix, Ep);
float x_offset_ = (Kep.v[0] / Kep.v[2] - intrinsic_matrix.v[2]) * zoom;
float y_offset_ = (Kep.v[1] / Kep.v[2] - intrinsic_matrix.v[5]) * zoom;
float max_x_offset = intrinsic_matrix.v[2] * zoom - w / 2 - 5;
float max_y_offset = intrinsic_matrix.v[5] * zoom - h / 2 - 5;
x_offset = std::clamp(x_offset_, -max_x_offset, max_x_offset);
y_offset = std::clamp(y_offset_, -max_y_offset, max_y_offset);
float zx = zoom * 2 * intrinsic_matrix.v[2] / w;
float zy = zoom * 2 * intrinsic_matrix.v[5] / h;
const mat4 frame_transform = {{
zx, 0.0, 0.0, -x_offset / w * 2,
0.0, zy, 0.0, y_offset / h * 2,
0.0, 0.0, 1.0, 0.0,
0.0, 0.0, 0.0, 1.0,
}};
frame_mat = frame_transform;
}
} else if (stream_width > 0 && stream_height > 0) {
// fit frame to widget size
float widget_aspect_ratio = (float)w / h;
float frame_aspect_ratio = (float)stream_width / stream_height;
frame_mat = get_fit_view_transform(widget_aspect_ratio, frame_aspect_ratio);
}
}
void CameraWidget::updateCalibration(const mat3 &calib) {
calibration = calib;
return mat4{{
zx, 0.0, 0.0, 0.0,
0.0, zy, 0.0, 0.0,
0.0, 0.0, 1.0, 0.0,
0.0, 0.0, 0.0, 1.0,
}};
}
void CameraWidget::paintGL() {
@ -293,7 +217,7 @@ void CameraWidget::paintGL() {
VisionBuf *frame = frames[frame_idx].second;
assert(frame != nullptr);
updateFrameMat();
auto frame_mat = calcFrameMatrix();
glViewport(0, 0, glWidth(), glHeight());
glBindVertexArray(frame_vao);

@ -34,7 +34,7 @@ class CameraWidget : public QOpenGLWidget, protected QOpenGLFunctions {
public:
using QOpenGLWidget::QOpenGLWidget;
explicit CameraWidget(std::string stream_name, VisionStreamType stream_type, bool zoom, QWidget* parent = nullptr);
explicit CameraWidget(std::string stream_name, VisionStreamType stream_type, QWidget* parent = nullptr);
~CameraWidget();
void setBackgroundColor(const QColor &color) { bg = color; }
void setFrameId(int frame_id) { draw_frame_id = frame_id; }
@ -51,21 +51,17 @@ signals:
protected:
void paintGL() override;
void initializeGL() override;
void resizeGL(int w, int h) override { updateFrameMat(); }
void showEvent(QShowEvent *event) override;
void mouseReleaseEvent(QMouseEvent *event) override { emit clicked(); }
virtual void updateFrameMat();
void updateCalibration(const mat3 &calib);
virtual mat4 calcFrameMatrix();
void vipcThread();
void clearFrames();
int glWidth();
int glHeight();
bool zoomed_view;
GLuint frame_vao, frame_vbo, frame_ibo;
GLuint textures[2];
mat4 frame_mat = {};
std::unique_ptr<QOpenGLShaderProgram> program;
QColor bg = QColor("#000000");
@ -81,14 +77,6 @@ protected:
std::atomic<VisionStreamType> requested_stream_type;
std::set<VisionStreamType> available_streams;
QThread *vipc_thread = nullptr;
// Calibration
float x_offset = 0;
float y_offset = 0;
float zoom = 1.0;
mat3 calibration = DEFAULT_CALIBRATION;
mat3 intrinsic_matrix = FCAM_INTRINSIC_MATRIX;
std::recursive_mutex frame_lock;
std::deque<std::pair<uint32_t, VisionBuf*>> frames;
uint32_t draw_frame_id = 0;

@ -11,7 +11,7 @@
const QString BACKSPACE_KEY = "";
const QString ENTER_KEY = "";
const QMap<QString, int> KEY_STRETCH = {{" ", 5}, {ENTER_KEY, 2}};
const QMap<QString, int> KEY_STRETCH = {{" ", 3}, {ENTER_KEY, 2}};
const QStringList CONTROL_BUTTONS = {"", "", "ABC", "#+=", "123", BACKSPACE_KEY, ENTER_KEY};
@ -107,7 +107,7 @@ Keyboard::Keyboard(QWidget *parent) : QFrame(parent) {
{"q", "w", "e", "r", "t", "y", "u", "i", "o", "p"},
{"a", "s", "d", "f", "g", "h", "j", "k", "l"},
{"", "z", "x", "c", "v", "b", "n", "m", BACKSPACE_KEY},
{"123", " ", ".", ENTER_KEY},
{"123", "/", "-", " ", ".", ENTER_KEY},
};
main_layout->addWidget(new KeyboardLayout(this, lowercase));

@ -246,33 +246,12 @@ SetupWidget::SetupWidget(QWidget* parent) : QFrame(parent) {
sp_retain.setRetainSizeWhenHidden(true);
setSizePolicy(sp_retain);
// set up API requests
if (auto dongleId = getDongleId()) {
QString url = CommaApi::BASE_URL + "/v1.1/devices/" + *dongleId + "/";
RequestRepeater* repeater = new RequestRepeater(this, url, "ApiCache_Device", 5);
QObject::connect(repeater, &RequestRepeater::requestDone, this, &SetupWidget::replyFinished);
}
}
void SetupWidget::replyFinished(const QString &response, bool success) {
if (!success) return;
QJsonDocument doc = QJsonDocument::fromJson(response.toUtf8());
if (doc.isNull()) {
qDebug() << "JSON Parse failed on getting pairing and prime status";
return;
}
QJsonObject json = doc.object();
bool is_paired = json["is_paired"].toBool();
PrimeType prime_type = static_cast<PrimeType>(json["prime_type"].toInt());
uiState()->setPrimeType(is_paired ? prime_type : PrimeType::PRIME_TYPE_UNPAIRED);
if (!is_paired) {
mainLayout->setCurrentIndex(0);
} else {
popup->reject();
mainLayout->setCurrentIndex(1);
}
QObject::connect(uiState()->prime_state, &PrimeState::changed, [this](PrimeState::Type type) {
if (type == PrimeState::PRIME_TYPE_UNPAIRED) {
mainLayout->setCurrentIndex(0); // Display "Pair your device" widget
} else {
popup->reject();
mainLayout->setCurrentIndex(1); // Display Wi-Fi prompt widget
}
});
}

@ -66,7 +66,4 @@ signals:
private:
PairingPopup *popup;
QStackedWidget *mainLayout;
private slots:
void replyFinished(const QString &response, bool success);
};

@ -169,6 +169,7 @@ class TestUI:
def setup(self):
self.pm = PubMaster(list(DATA.keys()))
DATA['deviceState'].deviceState.networkType = log.DeviceState.NetworkType.wifi
DATA['deviceState'].deviceState.lastAthenaPingTime = 0
for _ in range(10):
self.pm.send('deviceState', DATA['deviceState'])
DATA['deviceState'].clear_write_flag()

@ -419,11 +419,11 @@
</message>
<message>
<source>Waiting to start</source>
<translation type="unfinished"></translation>
<translation></translation>
</message>
<message>
<source>System Unresponsive</source>
<translation type="unfinished"></translation>
<translation></translation>
</message>
</context>
<context>
@ -476,7 +476,7 @@
</message>
<message>
<source>24/7 LTE connectivity</source>
<translation> LTE </translation>
<translation> LTE </translation>
</message>
<message>
<source>1 year of drive storage</source>
@ -484,7 +484,7 @@
</message>
<message>
<source>Remote snapshots</source>
<translation type="unfinished"></translation>
<translation></translation>
</message>
</context>
<context>

@ -419,11 +419,11 @@
</message>
<message>
<source>Waiting to start</source>
<translation type="unfinished"></translation>
<translation></translation>
</message>
<message>
<source>System Unresponsive</source>
<translation type="unfinished"></translation>
<translation></translation>
</message>
</context>
<context>
@ -484,7 +484,7 @@
</message>
<message>
<source>Remote snapshots</source>
<translation type="unfinished"></translation>
<translation></translation>
</message>
</context>
<context>

@ -1,13 +1,11 @@
#include "selfdrive/ui/ui.h"
#include <algorithm>
#include <cassert>
#include <cmath>
#include <QtConcurrent>
#include "common/transformations/orientation.hpp"
#include "common/params.h"
#include "common/swaglog.h"
#include "common/util.h"
#include "common/watchdog.h"
@ -19,20 +17,10 @@
// Projects a point in car to space to the corresponding point in full frame
// image space.
static bool calib_frame_to_full_frame(const UIState *s, float in_x, float in_y, float in_z, QPointF *out) {
const float margin = 500.0f;
const QRectF clip_region{-margin, -margin, s->fb_w + 2 * margin, s->fb_h + 2 * margin};
const vec3 pt = (vec3){{in_x, in_y, in_z}};
const vec3 Ep = matvecmul3(s->scene.wide_cam ? s->scene.view_from_wide_calib : s->scene.view_from_calib, pt);
const vec3 KEp = matvecmul3(s->scene.wide_cam ? ECAM_INTRINSIC_MATRIX : FCAM_INTRINSIC_MATRIX, Ep);
// Project.
QPointF point = s->car_space_transform.map(QPointF{KEp.v[0] / KEp.v[2], KEp.v[1] / KEp.v[2]});
if (clip_region.contains(point)) {
*out = point;
return true;
}
return false;
Eigen::Vector3f input(in_x, in_y, in_z);
auto transformed = s->car_space_transform * input;
*out = QPointF(transformed.x() / transformed.z(), transformed.y() / transformed.z());
return s->clip_region.contains(*out);
}
int get_path_length_idx(const cereal::XYZTData::Reader &line, const float path_height) {
@ -119,29 +107,19 @@ static void update_state(UIState *s) {
UIScene &scene = s->scene;
if (sm.updated("liveCalibration")) {
auto list2rot = [](const capnp::List<float>::Reader &rpy_list) ->Eigen::Matrix3f {
return euler2rot({rpy_list[0], rpy_list[1], rpy_list[2]}).cast<float>();
};
auto live_calib = sm["liveCalibration"].getLiveCalibration();
auto rpy_list = live_calib.getRpyCalib();
auto wfde_list = live_calib.getWideFromDeviceEuler();
Eigen::Vector3d rpy;
Eigen::Vector3d wfde;
if (rpy_list.size() == 3) rpy << rpy_list[0], rpy_list[1], rpy_list[2];
if (wfde_list.size() == 3) wfde << wfde_list[0], wfde_list[1], wfde_list[2];
Eigen::Matrix3d device_from_calib = euler2rot(rpy);
Eigen::Matrix3d wide_from_device = euler2rot(wfde);
Eigen::Matrix3d view_from_device;
view_from_device << 0, 1, 0,
0, 0, 1,
1, 0, 0;
Eigen::Matrix3d view_from_calib = view_from_device * device_from_calib;
Eigen::Matrix3d view_from_wide_calib = view_from_device * wide_from_device * device_from_calib;
for (int i = 0; i < 3; i++) {
for (int j = 0; j < 3; j++) {
scene.view_from_calib.v[i*3 + j] = view_from_calib(i, j);
scene.view_from_wide_calib.v[i*3 + j] = view_from_wide_calib(i, j);
}
if (live_calib.getCalStatus() == cereal::LiveCalibrationData::Status::CALIBRATED) {
auto device_from_calib = list2rot(live_calib.getRpyCalib());
auto wide_from_device = list2rot(live_calib.getWideFromDeviceEuler());
s->scene.view_from_calib = VIEW_FROM_DEVICE * device_from_calib;
s->scene.view_from_wide_calib = VIEW_FROM_DEVICE * wide_from_device * device_from_calib;
} else {
s->scene.view_from_calib = s->scene.view_from_wide_calib = VIEW_FROM_DEVICE;
}
scene.calibration_valid = live_calib.getCalStatus() == cereal::LiveCalibrationData::Status::CALIBRATED;
scene.calibration_wide_valid = wfde_list.size() == 3;
}
if (sm.updated("pandaStates")) {
auto pandaStates = sm["pandaStates"].getPandaStates();
@ -210,13 +188,8 @@ UIState::UIState(QObject *parent) : QObject(parent) {
"pandaStates", "carParams", "driverMonitoringState", "carState", "driverStateV2",
"wideRoadCameraState", "managerState", "selfdriveState",
});
Params params;
language = QString::fromStdString(params.get("LanguageSetting"));
auto prime_value = params.get("PrimeType");
if (!prime_value.empty()) {
prime_type = static_cast<PrimeType>(std::atoi(prime_value.c_str()));
}
prime_state = new PrimeState(this);
language = QString::fromStdString(Params().get("LanguageSetting"));
// update timer
timer = new QTimer(this);
@ -229,31 +202,12 @@ void UIState::update() {
update_state(this);
updateStatus();
if (std::getenv("PRIME_TYPE")) {
setPrimeType((PrimeType)atoi(std::getenv("PRIME_TYPE")));
}
if (sm->frame % UI_FREQ == 0) {
watchdog_kick(nanos_since_boot());
}
emit uiUpdate(*this);
}
void UIState::setPrimeType(PrimeType type) {
if (type != prime_type) {
bool prev_prime = hasPrime();
prime_type = type;
Params().put("PrimeType", std::to_string(prime_type));
emit primeTypeChanged(prime_type);
bool prime = hasPrime();
if (prev_prime != prime) {
emit primeChanged(prime);
}
}
}
Device::Device(QObject *parent) : brightness_filter(BACKLIGHT_OFFROAD, BACKLIGHT_TS, BACKLIGHT_DT), QObject(parent) {
setAwake(true);
resetInteractiveTimeout();

@ -1,20 +1,20 @@
#pragma once
#include <eigen3/Eigen/Dense>
#include <memory>
#include <string>
#include <QObject>
#include <QTimer>
#include <QColor>
#include <QFuture>
#include <QPolygonF>
#include <QTransform>
#include "cereal/messaging/messaging.h"
#include "common/mat.h"
#include "common/params.h"
#include "common/timing.h"
#include "common/util.h"
#include "system/hardware/hw.h"
#include "selfdrive/ui/qt/prime_state.h"
const int UI_BORDER_SIZE = 30;
const int UI_HEADER_HEIGHT = 420;
@ -24,15 +24,22 @@ const int BACKLIGHT_OFFROAD = 50;
const float MIN_DRAW_DISTANCE = 10.0;
const float MAX_DRAW_DISTANCE = 100.0;
constexpr mat3 DEFAULT_CALIBRATION = {{ 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0 }};
constexpr mat3 FCAM_INTRINSIC_MATRIX = (mat3){{2648.0, 0.0, 1928.0 / 2,
0.0, 2648.0, 1208.0 / 2,
0.0, 0.0, 1.0}};
const Eigen::Matrix3f VIEW_FROM_DEVICE = (Eigen::Matrix3f() <<
0.0, 1.0, 0.0,
0.0, 0.0, 1.0,
1.0, 0.0, 0.0).finished();
const Eigen::Matrix3f FCAM_INTRINSIC_MATRIX = (Eigen::Matrix3f() <<
2648.0, 0.0, 1928.0 / 2,
0.0, 2648.0, 1208.0 / 2,
0.0, 0.0, 1.0).finished();
// tici ecam focal probably wrong? magnification is not consistent across frame
// Need to retrain model before this can be changed
constexpr mat3 ECAM_INTRINSIC_MATRIX = (mat3){{567.0, 0.0, 1928.0 / 2,
0.0, 567.0, 1208.0 / 2,
0.0, 0.0, 1.0}};
const Eigen::Matrix3f ECAM_INTRINSIC_MATRIX = (Eigen::Matrix3f() <<
567.0, 0.0, 1928.0 / 2,
0.0, 567.0, 1208.0 / 2,
0.0, 0.0, 1.0).finished();
typedef enum UIStatus {
STATUS_DISENGAGED,
@ -40,30 +47,15 @@ typedef enum UIStatus {
STATUS_ENGAGED,
} UIStatus;
enum PrimeType {
PRIME_TYPE_UNKNOWN = -2,
PRIME_TYPE_UNPAIRED = -1,
PRIME_TYPE_NONE = 0,
PRIME_TYPE_MAGENTA = 1,
PRIME_TYPE_LITE = 2,
PRIME_TYPE_BLUE = 3,
PRIME_TYPE_MAGENTA_NEW = 4,
PRIME_TYPE_PURPLE = 5,
};
const QColor bg_colors [] = {
[STATUS_DISENGAGED] = QColor(0x17, 0x33, 0x49, 0xc8),
[STATUS_OVERRIDE] = QColor(0x91, 0x9b, 0x95, 0xf1),
[STATUS_ENGAGED] = QColor(0x17, 0x86, 0x44, 0xf1),
};
typedef struct UIScene {
bool calibration_valid = false;
bool calibration_wide_valid = false;
bool wide_cam = true;
mat3 view_from_calib = DEFAULT_CALIBRATION;
mat3 view_from_wide_calib = DEFAULT_CALIBRATION;
Eigen::Matrix3f view_from_calib = VIEW_FROM_DEVICE;
Eigen::Matrix3f view_from_wide_calib = VIEW_FROM_DEVICE;
cereal::PandaState::PandaType pandaType;
// modelV2
@ -94,26 +86,19 @@ public:
return scene.started && (*sm)["selfdriveState"].getSelfdriveState().getEnabled();
}
void setPrimeType(PrimeType type);
inline PrimeType primeType() const { return prime_type; }
inline bool hasPrime() const { return prime_type > PrimeType::PRIME_TYPE_NONE; }
int fb_w = 0, fb_h = 0;
std::unique_ptr<SubMaster> sm;
UIStatus status;
UIScene scene = {};
QString language;
PrimeState *prime_state;
QTransform car_space_transform;
Eigen::Matrix3f car_space_transform = Eigen::Matrix3f::Zero();
QRectF clip_region;
signals:
void uiUpdate(const UIState &s);
void offroadTransition(bool offroad);
void primeChanged(bool prime);
void primeTypeChanged(PrimeType prime_type);
private slots:
void update();
@ -121,7 +106,6 @@ private slots:
private:
QTimer *timer;
bool started_prev = false;
PrimeType prime_type = PrimeType::PRIME_TYPE_UNKNOWN;
};
UIState *uiState();
@ -166,7 +150,6 @@ void ui_update_params(UIState *s);
int get_path_length_idx(const cereal::XYZTData::Reader &line, const float path_height);
void update_model(UIState *s,
const cereal::ModelDataV2::Reader &model);
void update_dmonitoring(UIState *s, const cereal::DriverStateV2::Reader &driverstate, float dm_fade_state, bool is_rhd);
void update_leads(UIState *s, const cereal::RadarState::Reader &radar_state, const cereal::XYZTData::Reader &line);
void update_line_data(const UIState *s, const cereal::XYZTData::Reader &line,
float y_off, float z_off, QPolygonF *pvd, int max_idx, bool allow_invert);

@ -19,14 +19,14 @@ int main(int argc, char *argv[]) {
{
QHBoxLayout *hlayout = new QHBoxLayout();
layout->addLayout(hlayout);
hlayout->addWidget(new CameraWidget("camerad", VISION_STREAM_ROAD, false));
hlayout->addWidget(new CameraWidget("camerad", VISION_STREAM_ROAD));
}
{
QHBoxLayout *hlayout = new QHBoxLayout();
layout->addLayout(hlayout);
hlayout->addWidget(new CameraWidget("camerad", VISION_STREAM_DRIVER, false));
hlayout->addWidget(new CameraWidget("camerad", VISION_STREAM_WIDE_ROAD, false));
hlayout->addWidget(new CameraWidget("camerad", VISION_STREAM_DRIVER));
hlayout->addWidget(new CameraWidget("camerad", VISION_STREAM_WIDE_ROAD));
}
return a.exec();

@ -50,7 +50,7 @@ class TestStreamSession:
tested_msgs = [
{"type": "customReservedRawData0", "data": "test"}, # primitive
{"type": "can", "data": [{"address": 0, "dat": "", "src": 0}]}, # list
{"type": "testJoystick", "data": {"axes": [0, 0], "buttons": [False]}}, # dict
{"type": "testJoystickDEPRECATED", "data": {"axes": [0, 0], "buttons": [False]}}, # dict
]
mocked_pubmaster = mocker.MagicMock(spec=messaging.PubMaster)

@ -1 +1 @@
Subproject commit ae5d1407ee844a97a52ad3756835d38e7e2b9e1b
Subproject commit f51aa0fc7cdbac710e640172db280cfb747d2718

@ -148,7 +148,7 @@ QWidget *VideoWidget::createCameraWidget() {
QStackedLayout *stacked = new QStackedLayout();
stacked->setStackingMode(QStackedLayout::StackAll);
stacked->addWidget(cam_widget = new StreamCameraView("camerad", VISION_STREAM_ROAD, false));
stacked->addWidget(cam_widget = new StreamCameraView("camerad", VISION_STREAM_ROAD));
cam_widget->setMinimumHeight(MIN_VIDEO_HEIGHT);
cam_widget->setSizePolicy(QSizePolicy::Preferred, QSizePolicy::MinimumExpanding);
stacked->addWidget(alert_label = new InfoLabel(this));
@ -420,8 +420,8 @@ void InfoLabel::paintEvent(QPaintEvent *event) {
}
}
StreamCameraView::StreamCameraView(std::string stream_name, VisionStreamType stream_type, bool zoom, QWidget *parent)
: CameraWidget(stream_name, stream_type, zoom, parent) {
StreamCameraView::StreamCameraView(std::string stream_name, VisionStreamType stream_type, QWidget *parent)
: CameraWidget(stream_name, stream_type, parent) {
fade_animation = new QPropertyAnimation(this, "overlayOpacity");
fade_animation->setDuration(500);
fade_animation->setStartValue(0.2f);

@ -64,7 +64,7 @@ class StreamCameraView : public CameraWidget {
Q_PROPERTY(float overlayOpacity READ overlayOpacity WRITE setOverlayOpacity)
public:
StreamCameraView(std::string stream_name, VisionStreamType stream_type, bool zoom, QWidget *parent = nullptr);
StreamCameraView(std::string stream_name, VisionStreamType stream_type, QWidget *parent = nullptr);
void paintGL() override;
void showPausedOverlay() { fade_animation->start(); }
float overlayOpacity() const { return overlay_opacity; }

Loading…
Cancel
Save