cabana: dispaly alert info in thumbnail&video player (#27837)

* show alert info in thumbnail

* dispaly alert info in video
old-commit-hash: 2d061e2041

tools/cabana/videowidget.cc

@@ -3,6 +3,7 @@
 #include <QButtonGroup>
 #include <QMouseEvent>
 #include <QPainter>
+#include <QStackedLayout>
 #include <QStyleOptionSlider>
 #include <QVBoxLayout>
 #include <QtConcurrent>
@@ -19,7 +20,6 @@ static const QColor timeline_colors[] = {
   [(int)TimelineType::AlertCritical] = QColor(199, 0, 57),
 };
 
-
 bool sortTimelineBasedOnEventPriority(const std::tuple<int, int, TimelineType> &left, const std::tuple<int, int, TimelineType> &right){
   const static std::map<TimelineType, int> timelinePriority = {
     {  TimelineType::None, 0 },
@@ -29,7 +29,6 @@ bool sortTimelineBasedOnEventPriority(const std::tuple<int, int, TimelineType> &
     {  TimelineType::AlertCritical, 40 },
     {  TimelineType::UserFlag, 35 }
   };
-
   return timelinePriority.at(std::get<2>(left)) < timelinePriority.at(std::get<2>(right));
 }
 
@@ -92,10 +91,14 @@ QWidget *VideoWidget::createCameraWidget() {
   QWidget *w = new QWidget(this);
   QVBoxLayout *l = new QVBoxLayout(w);
   l->setContentsMargins(0, 0, 0, 0);
-  cam_widget = new CameraWidget("camerad", can->visionStreamType(), false);
+
+  QStackedLayout *stacked = new QStackedLayout();
+  stacked->setStackingMode(QStackedLayout::StackAll);
+  stacked->addWidget(cam_widget = new CameraWidget("camerad", can->visionStreamType(), false));
   cam_widget->setMinimumHeight(MIN_VIDEO_HEIGHT);
   cam_widget->setSizePolicy(QSizePolicy::Preferred, QSizePolicy::MinimumExpanding);
-  l->addWidget(cam_widget);
+  stacked->addWidget(alert_label = new InfoLabel(this));
+  l->addLayout(stacked);
 
   // slider controls
   slider_layout = new QHBoxLayout();
@@ -130,9 +133,18 @@ void VideoWidget::rangeChanged(double min, double max, bool is_zoomed) {
 }
 
 void VideoWidget::updateState() {
-  if (!slider->isSliderDown())
+  if (!slider->isSliderDown()) {
     slider->setValue(can->currentSec() * 1000);
   }
+  std::lock_guard lk(slider->thumbnail_lock);
+  uint64_t mono_time = (can->currentSec() + can->routeStartTime()) * 1e9;
+  auto it = slider->alerts.lower_bound(mono_time);
+  if (it != slider->alerts.end() && (it->first - mono_time) < 1e9) {
+    alert_label->showAlert(it->second);
+  } else {
+    alert_label->showAlert({});
+  }
+}
 
 void VideoWidget::updatePlayBtnState() {
   play_btn->setIcon(utils::icon(can->isPaused() ? "play" : "pause"));
@@ -144,7 +156,6 @@ Slider::Slider(QWidget *parent) : timer(this), thumbnail_label(this), QSlider(Qt
   timer.callOnTimeout([this]() {
     timeline = can->getTimeline();
     std::sort(timeline.begin(), timeline.end(), sortTimelineBasedOnEventPriority);
-
     update();
   });
   setMouseTracking(true);
@@ -169,11 +180,11 @@ void Slider::streamStarted() {
 void Slider::loadThumbnails() {
   const auto &segments = can->route()->segments();
   for (auto it = segments.rbegin(); it != segments.rend() && !abort_load_thumbnail; ++it) {
-    std::string qlog = it->second.qlog.toStdString();
-    if (!qlog.empty()) {
     LogReader log;
-      if (log.load(qlog, &abort_load_thumbnail, {cereal::Event::Which::THUMBNAIL}, true, 0, 3)) {
+    std::string qlog = it->second.qlog.toStdString();
+    if (!qlog.empty() && log.load(qlog, &abort_load_thumbnail, {cereal::Event::Which::THUMBNAIL, cereal::Event::Which::CONTROLS_STATE}, true, 0, 3)) {
       for (auto ev = log.events.cbegin(); ev != log.events.cend() && !abort_load_thumbnail; ++ev) {
+        if ((*ev)->which == cereal::Event::Which::THUMBNAIL) {
           auto thumb = (*ev)->event.getThumbnail();
           auto data = thumb.getThumbnail();
           if (QPixmap pm; pm.loadFromData(data.begin(), data.size(), "jpeg")) {
@@ -181,6 +192,12 @@ void Slider::loadThumbnails() {
             std::lock_guard lk(thumbnail_lock);
             thumbnails[thumb.getTimestampEof()] = pm;
           }
+        } else if ((*ev)->which == cereal::Event::Which::CONTROLS_STATE) {
+          auto cs = (*ev)->event.getControlsState();
+          if (cs.getAlertType().size() > 0 && cs.getAlertText1().size() > 0) {
+            std::lock_guard lk(thumbnail_lock);
+            alerts.emplace((*ev)->mono_time, AlertInfo{cs.getAlertStatus(), cs.getAlertText1().cStr(), cs.getAlertText2().cStr()});
+          }
         }
       }
     }
@@ -234,15 +251,21 @@ void Slider::mousePressEvent(QMouseEvent *e) {
 
 void Slider::mouseMoveEvent(QMouseEvent *e) {
   QPixmap thumb;
+  AlertInfo alert;
   double seconds = (minimum() + e->pos().x() * ((maximum() - minimum()) / (double)width())) / 1000.0;
   {
     std::lock_guard lk(thumbnail_lock);
-    auto it = thumbnails.lowerBound((seconds + can->routeStartTime()) * 1e9);
+    uint64_t mono_time = (seconds + can->routeStartTime()) * 1e9;
+    auto it = thumbnails.lowerBound(mono_time);
     if (it != thumbnails.end()) thumb = it.value();
+    auto alert_it = alerts.lower_bound(mono_time);
+    if (alert_it != alerts.end() && (alert_it->first - mono_time) < 1e9) {
+      alert = alert_it->second;
+    }
   }
   int x = std::clamp(e->pos().x() - thumb.width() / 2, THUMBNAIL_MARGIN, rect().right() - thumb.width() - THUMBNAIL_MARGIN);
   int y = -thumb.height() - THUMBNAIL_MARGIN - style()->pixelMetric(QStyle::PM_LayoutVerticalSpacing);
-  thumbnail_label.showPixmap(mapToGlobal({x, y}), utils::formatSeconds(seconds), thumb);
+  thumbnail_label.showPixmap(mapToGlobal({x, y}), utils::formatSeconds(seconds), thumb, alert);
   QSlider::mouseMoveEvent(e);
 }
 
@@ -251,16 +274,17 @@ void Slider::leaveEvent(QEvent *event) {
   QSlider::leaveEvent(event);
 }
 
-// ThumbnailLabel
+// InfoLabel
 
-ThumbnailLabel::ThumbnailLabel(QWidget *parent) : QWidget(parent, Qt::Tool | Qt::FramelessWindowHint) {
+InfoLabel::InfoLabel(QWidget *parent) : QWidget(parent, Qt::Tool | Qt::FramelessWindowHint) {
   setAttribute(Qt::WA_ShowWithoutActivating);
   setVisible(false);
 }
 
-void ThumbnailLabel::showPixmap(const QPoint &pt, const QString &sec, const QPixmap &pm) {
+void InfoLabel::showPixmap(const QPoint &pt, const QString &sec, const QPixmap &pm, const AlertInfo &alert) {
   pixmap = pm;
   second = sec;
+  alert_info = alert;
   setVisible(!pm.isNull());
   if (isVisible()) {
     setGeometry({pt, pm.size()});
@@ -268,10 +292,42 @@ void ThumbnailLabel::showPixmap(const QPoint &pt, const QString &sec, const QPix
   }
 }
 
-void ThumbnailLabel::paintEvent(QPaintEvent *event) {
+void InfoLabel::showAlert(const AlertInfo &alert) {
+  alert_info = alert;
+  pixmap = {};
+  setVisible(!alert_info.text1.isEmpty());
+  if (isVisible()) {
+    update();
+  }
+}
+
+void InfoLabel::paintEvent(QPaintEvent *event) {
   QPainter p(this);
-  p.drawPixmap(0, 0, pixmap);
   p.setPen(QPen(Qt::white, 2));
+  if (!pixmap.isNull()) {
+    p.drawPixmap(0, 0, pixmap);
     p.drawRect(rect());
     p.drawText(rect().adjusted(0, 0, 0, -THUMBNAIL_MARGIN), second, Qt::AlignHCenter | Qt::AlignBottom);
   }
+  if (alert_info.text1.size() > 0) {
+    QColor color = timeline_colors[(int)TimelineType::AlertInfo];
+    if (alert_info.status == cereal::ControlsState::AlertStatus::USER_PROMPT) {
+      color = timeline_colors[(int)TimelineType::AlertWarning];
+    } else if (alert_info.status == cereal::ControlsState::AlertStatus::CRITICAL) {
+      color = timeline_colors[(int)TimelineType::AlertCritical];
+    }
+    color.setAlphaF(0.5);
+    QString text = alert_info.text1;
+    if (!alert_info.text2.isEmpty()) {
+      text += "\n" + alert_info.text2;
+    }
+
+    QFont font;
+    font.setPixelSize(!pixmap.isNull() ? 11 : QFont().pixelSize());
+    p.setFont(font);
+    QRect text_rect = rect().adjusted(2, 2, -2, -2);
+    QRect r = p.fontMetrics().boundingRect(text_rect, Qt::AlignTop | Qt::AlignHCenter | Qt::TextWordWrap, text);
+    p.fillRect(text_rect.left(), r.top(), text_rect.width(), r.height(), color);
+    p.drawText(text_rect, Qt::AlignTop | Qt::AlignHCenter | Qt::TextWordWrap, text);
+  }
+}

tools/cabana/videowidget.h

@@ -13,13 +13,21 @@
 #include "selfdrive/ui/qt/widgets/cameraview.h"
 #include "tools/cabana/streams/abstractstream.h"
 
-class ThumbnailLabel : public QWidget {
+struct AlertInfo {
+  cereal::ControlsState::AlertStatus status;
+  QString text1;
+  QString text2;
+};
+
+class InfoLabel : public QWidget {
 public:
-  ThumbnailLabel(QWidget *parent);
+  InfoLabel(QWidget *parent);
-  void showPixmap(const QPoint &pt, const QString &sec, const QPixmap &pm);
+  void showPixmap(const QPoint &pt, const QString &sec, const QPixmap &pm, const AlertInfo &alert);
+  void showAlert(const AlertInfo &alert);
   void paintEvent(QPaintEvent *event) override;
   QPixmap pixmap;
   QString second;
+  AlertInfo alert_info;
 };
 
 class Slider : public QSlider {
@@ -43,9 +51,11 @@ private:
   std::mutex thumbnail_lock;
   std::atomic<bool> abort_load_thumbnail = false;
   QMap<uint64_t, QPixmap> thumbnails;
+  std::map<uint64_t, AlertInfo> alerts;
   QFuture<void> thumnail_future;
-  ThumbnailLabel thumbnail_label;
+  InfoLabel thumbnail_label;
   QTimer timer;
+  friend class VideoWidget;
 };
 
 class VideoWidget : public QFrame {
@@ -65,5 +75,6 @@ protected:
   QLabel *time_label;
   QHBoxLayout *slider_layout;
   QPushButton *play_btn;
+  InfoLabel *alert_label;
   Slider *slider;
 };