cabana: real-time cursor and video frame sync for chart and video (#34301)

* sync cursor and thumbnail between chart and video

* Revert "replay: Update video immediately after seek when paused. (#34237)"

This reverts commit 3363881844.

* use thumbnails while scrubing

* draw alert

* no update on resume

* draw timestamp

* cleanup

tools/cabana/chart/chartswidget.cc

@@ -193,6 +193,7 @@ QRect ChartsWidget::chartVisibleRect(ChartView *chart) {
 }
 
 void ChartsWidget::showValueTip(double sec) {
+  emit showTip(sec);
   if (sec < 0 && !value_tip_visible_) return;
 
   value_tip_visible_ = sec >= 0;
@@ -548,20 +549,14 @@ void ChartsContainer::dropEvent(QDropEvent *event) {
 
 void ChartsContainer::paintEvent(QPaintEvent *ev) {
   if (!drop_indictor_pos.isNull() && !childAt(drop_indictor_pos)) {
-    QRect r;
+    QRect r = geometry();
+    r.setHeight(CHART_SPACING);
     if (auto insert_after = getDropAfter(drop_indictor_pos)) {
-      QRect area = insert_after->geometry();
+      r.moveTop(insert_after->geometry().bottom());
-      r = QRect(area.left(), area.bottom() + 1, area.width(), CHART_SPACING);
-    } else {
-      r = geometry();
-      r.setHeight(CHART_SPACING);
     }
 
     QPainter p(this);
-    p.setPen(QPen(palette().highlight(), 2));
+    p.fillRect(r, palette().highlight());
-    p.drawLine(r.topLeft() + QPoint(1, 0), r.bottomLeft() + QPoint(1, 0));
-    p.drawLine(r.topLeft() + QPoint(0, r.height() / 2), r.topRight() + QPoint(0, r.height() / 2));
-    p.drawLine(r.topRight(), r.bottomRight());
   }
 }
 

tools/cabana/chart/chartswidget.h

@@ -53,6 +53,7 @@ public slots:
 signals:
   void toggleChartsDocking();
   void seriesChanged();
+  void showTip(double seconds);
 
 private:
   QSize minimumSizeHint() const override;

tools/cabana/mainwin.cc

@@ -191,6 +191,7 @@ void MainWindow::createDockWidgets() {
   video_splitter->handle(1)->setEnabled(!can->liveStreaming());
   video_dock->setWidget(video_splitter);
   QObject::connect(charts_widget, &ChartsWidget::toggleChartsDocking, this, &MainWindow::toggleChartsDocking);
+  QObject::connect(charts_widget, &ChartsWidget::showTip, video_widget, &VideoWidget::showThumbnail);
 }
 
 void MainWindow::createStatusBar() {

tools/cabana/videowidget.cc

@@ -152,7 +152,6 @@ QWidget *VideoWidget::createCameraWidget() {
 
   QObject::connect(slider, &QSlider::sliderReleased, [this]() { can->seekTo(slider->currentSecond()); });
   QObject::connect(can, &AbstractStream::paused, cam_widget, [c = cam_widget]() { c->showPausedOverlay(); });
-  QObject::connect(can, &AbstractStream::resume, cam_widget, [c = cam_widget]() { c->update(); });
   QObject::connect(can, &AbstractStream::eventsMerged, this, [this]() { slider->update(); });
   QObject::connect(cam_widget, &CameraWidget::clicked, []() { can->pause(!can->isPaused()); });
   QObject::connect(cam_widget, &CameraWidget::vipcAvailableStreamsUpdated, this, &VideoWidget::vipcAvailableStreamsUpdated);
@@ -160,7 +159,7 @@ QWidget *VideoWidget::createCameraWidget() {
     if (index != -1) cam_widget->setStreamType((VisionStreamType)camera_tab->tabData(index).toInt());
   });
   QObject::connect(static_cast<ReplayStream*>(can), &ReplayStream::qLogLoaded, cam_widget, &StreamCameraView::parseQLog, Qt::QueuedConnection);
-  slider->installEventFilter(cam_widget);
+  slider->installEventFilter(this);
   return w;
 }
 
@@ -222,8 +221,24 @@ void VideoWidget::updatePlayBtnState() {
   play_toggle_action->setToolTip(can->isPaused() ? tr("Play") : tr("Pause"));
 }
 
-// Slider
+void VideoWidget::showThumbnail(double seconds) {
+  cam_widget->thumbnail_dispaly_time = seconds;
+  slider->thumbnail_dispaly_time = seconds;
+  cam_widget->update();
+  slider->update();
+}
+
+bool VideoWidget::eventFilter(QObject *obj, QEvent *event) {
+  if (event->type() == QEvent::MouseMove) {
+    auto [min_sec, max_sec] = can->timeRange().value_or(std::make_pair(can->minSeconds(), can->maxSeconds()));
+    showThumbnail(min_sec + static_cast<QMouseEvent *>(event)->pos().x() * (max_sec - min_sec) / slider->width());
+  } else if (event->type() == QEvent::Leave) {
+    showThumbnail(-1);
+  }
+  return false;
+}
 
+// Slider
 Slider::Slider(QWidget *parent) : QSlider(Qt::Horizontal, parent) {
   setMouseTracking(true);
 }
@@ -265,6 +280,14 @@ void Slider::paintEvent(QPaintEvent *ev) {
   opt.subControls = QStyle::SC_SliderHandle;
   opt.sliderPosition = value();
   style()->drawComplexControl(QStyle::CC_Slider, &opt, &p);
+
+  if (thumbnail_dispaly_time >= 0) {
+    int left = (thumbnail_dispaly_time - min) * width() / (max - min) - 1;
+    QRect rc(left, rect().top() + 1, 2, rect().height() - 2);
+    p.setBrush(palette().highlight());
+    p.setPen(Qt::NoPen);
+    p.drawRoundedRect(rc, 1.5, 1.5);
+  }
 }
 
 void Slider::mousePressEvent(QMouseEvent *e) {
@@ -276,7 +299,6 @@ void Slider::mousePressEvent(QMouseEvent *e) {
 }
 
 // StreamCameraView
-
 StreamCameraView::StreamCameraView(std::string stream_name, VisionStreamType stream_type, QWidget *parent)
     : CameraWidget(stream_name, stream_type, parent) {
   fade_animation = new QPropertyAnimation(this, "overlayOpacity");
@@ -298,6 +320,7 @@ void StreamCameraView::parseQLog(std::shared_ptr<LogReader> qlog) {
         QPixmap generated_thumb = generateThumbnail(thumb, can->toSeconds(thumb_data.getTimestampEof()));
         std::lock_guard lock(mutex);
         thumbnails[thumb_data.getTimestampEof()] = generated_thumb;
+        big_thumbnails[thumb_data.getTimestampEof()] = thumb;
       }
     }
   });
@@ -308,12 +331,15 @@ void StreamCameraView::paintGL() {
   CameraWidget::paintGL();
 
   QPainter p(this);
-  if (auto alert = getReplay()->findAlertAtTime(can->currentSec())) {
+  bool scrubbing = false;
-    drawAlert(p, rect(), *alert);
+  if (thumbnail_dispaly_time >= 0) {
+    scrubbing = can->isPaused();
+    scrubbing ? drawScrubThumbnail(p) : drawThumbnail(p);
   }
-  if (thumbnail_pt_) {
+  if (auto alert = getReplay()->findAlertAtTime(scrubbing ? thumbnail_dispaly_time : can->currentSec())) {
-    drawThumbnail(p);
+    drawAlert(p, rect(), *alert);
   }
+
   if (can->isPaused()) {
     p.setPen(QColor(200, 200, 200, static_cast<int>(255 * fade_animation->currentValue().toFloat())));
     p.setFont(QFont(font().family(), 16, QFont::Bold));
@@ -333,25 +359,37 @@ QPixmap StreamCameraView::generateThumbnail(QPixmap thumb, double seconds) {
   return scaled;
 }
 
-void StreamCameraView::drawThumbnail(QPainter &p) {
+void StreamCameraView::drawScrubThumbnail(QPainter &p) {
-  int pos = std::clamp(thumbnail_pt_->x(), 0, width());
+  p.fillRect(rect(), Qt::black);
-  auto [min_sec, max_sec] = can->timeRange().value_or(std::make_pair(can->minSeconds(), can->maxSeconds()));
+  auto it = big_thumbnails.lowerBound(can->toMonoTime(thumbnail_dispaly_time));
-  double seconds = min_sec + pos * (max_sec - min_sec) / width();
+  if (it != big_thumbnails.end()) {
+    QPixmap scaled_thumb = it.value().scaled(rect().size(), Qt::KeepAspectRatio, Qt::SmoothTransformation);
+    QRect thumb_rect(rect().center() - scaled_thumb.rect().center(), scaled_thumb.size());
+    p.drawPixmap(thumb_rect.topLeft(), scaled_thumb);
+    drawTime(p, thumb_rect, thumbnail_dispaly_time);
+  }
+}
 
-  auto it = thumbnails.lowerBound(can->toMonoTime(seconds));
+void StreamCameraView::drawThumbnail(QPainter &p) {
+  auto it = thumbnails.lowerBound(can->toMonoTime(thumbnail_dispaly_time));
   if (it != thumbnails.end()) {
     const QPixmap &thumb = it.value();
+    auto [min_sec, max_sec] = can->timeRange().value_or(std::make_pair(can->minSeconds(), can->maxSeconds()));
+    int pos = (thumbnail_dispaly_time - min_sec) * width() / (max_sec - min_sec);
     int x = std::clamp(pos - thumb.width() / 2, THUMBNAIL_MARGIN, width() - thumb.width() - THUMBNAIL_MARGIN + 1);
     int y = height() - thumb.height() - THUMBNAIL_MARGIN;
 
     p.drawPixmap(x, y, thumb);
-    p.setPen(QPen(palette().color(QPalette::BrightText), 2));
+    drawTime(p, QRect{x, y, thumb.width(), thumb.height()}, thumbnail_dispaly_time);
-    p.setFont(QFont(font().family(), 10));
-    p.drawText(x, y, thumb.width(), thumb.height() - THUMBNAIL_MARGIN,
-               Qt::AlignHCenter | Qt::AlignBottom, QString::number(seconds, 'f', 3));
   }
 }
 
+void StreamCameraView::drawTime(QPainter &p, const QRect &rect, double seconds) {
+  p.setPen(palette().color(QPalette::BrightText));
+  p.setFont(QFont(font().family(), 10));
+  p.drawText(rect.adjusted(0, 0, 0, -THUMBNAIL_MARGIN), Qt::AlignHCenter | Qt::AlignBottom, QString::number(seconds, 'f', 3));
+}
+
 void StreamCameraView::drawAlert(QPainter &p, const QRect &rect, const Timeline::Entry &alert) {
   p.setPen(QPen(palette().color(QPalette::BrightText), 2));
   QColor color = timeline_colors[int(alert.type)];
@@ -364,14 +402,3 @@ void StreamCameraView::drawAlert(QPainter &p, const QRect &rect, const Timeline:
   p.fillRect(text_rect.left(), r.top(), text_rect.width(), r.height(), color);
   p.drawText(text_rect, Qt::AlignTop | Qt::AlignHCenter | Qt::TextWordWrap, text);
 }
-
-bool StreamCameraView::eventFilter(QObject *, QEvent *event) {
-  if (event->type() == QEvent::MouseMove) {
-    thumbnail_pt_ = static_cast<QMouseEvent *>(event)->pos();
-    update();
-  } else if (event->type() == QEvent::Leave) {
-    thumbnail_pt_ = std::nullopt;
-    update();
-  }
-  return false;
-}

tools/cabana/videowidget.h

@@ -1,7 +1,6 @@
 #pragma once
 
 #include <memory>
-#include <optional>
 #include <set>
 #include <string>
 #include <utility>
@@ -28,6 +27,7 @@ public:
   void mousePressEvent(QMouseEvent *e) override;
   void paintEvent(QPaintEvent *ev) override;
   const double factor = 1000.0;
+  double thumbnail_dispaly_time = -1;
 };
 
 class StreamCameraView : public CameraWidget {
@@ -43,11 +43,14 @@ private:
   QPixmap generateThumbnail(QPixmap thumbnail, double seconds);
   void drawAlert(QPainter &p, const QRect &rect, const Timeline::Entry &alert);
   void drawThumbnail(QPainter &p);
-  bool eventFilter(QObject *obj, QEvent *event) override;
+  void drawScrubThumbnail(QPainter &p);
+  void drawTime(QPainter &p, const QRect &rect, double seconds);
 
   QPropertyAnimation *fade_animation;
+  QMap<uint64_t, QPixmap> big_thumbnails;
   QMap<uint64_t, QPixmap> thumbnails;
-  std::optional<QPoint> thumbnail_pt_;
+  double thumbnail_dispaly_time = -1;
+  friend class VideoWidget;
 };
 
 class VideoWidget : public QFrame {
@@ -55,8 +58,10 @@ class VideoWidget : public QFrame {
 
 public:
   VideoWidget(QWidget *parnet = nullptr);
+  void showThumbnail(double seconds);
 
 protected:
+  bool eventFilter(QObject *obj, QEvent *event) override;
   QString formatTime(double sec, bool include_milliseconds = false);
   void timeRangeChanged();
   void updateState();

tools/replay/replay.cc

@@ -89,7 +89,7 @@ void Replay::interruptStream(const std::function<bool()> &update_fn) {
     interrupt_requested_ = true;
     std::unique_lock lock(stream_lock_);
     events_ready_ = update_fn();
-    interrupt_requested_ = user_paused_ && !pause_after_next_frame_;
+    interrupt_requested_ = user_paused_;
   }
   stream_cv_.notify_one();
 }
@@ -116,9 +116,6 @@ void Replay::seekTo(double seconds, bool relative) {
       seeked_to_sec = *seeking_to_;
       seeking_to_.reset();
     }
-
-    // if paused, resume for exactly one frame to update
-    pause_after_next_frame_ = user_paused_;
     return false;
   });
 
@@ -147,7 +144,6 @@ void Replay::pause(bool pause) {
     interruptStream([=]() {
       rWarning("%s at %.2f s", pause ? "paused..." : "resuming", currentSeconds());
       user_paused_ = pause;
-      pause_after_next_frame_ = false;
       return !pause;
     });
   }
@@ -309,7 +305,6 @@ std::vector<Event>::const_iterator Replay::publishEvents(std::vector<Event>::con
   uint64_t evt_start_ts = cur_mono_time_;
   uint64_t loop_start_ts = nanos_since_boot();
   double prev_replay_speed = speed_;
-  uint64_t first_mono_time = first->mono_time;
 
   for (; !interrupt_requested_ && first != last; ++first) {
     const Event &evt = *first;
@@ -348,12 +343,6 @@ std::vector<Event>::const_iterator Replay::publishEvents(std::vector<Event>::con
       }
       publishFrame(&evt);
     }
-
-    const auto T_ONE_FRAME = 0.050;
-    if (pause_after_next_frame_ && abs(toSeconds(evt.mono_time) - toSeconds(first_mono_time)) > T_ONE_FRAME) {
-      pause_after_next_frame_ = false;
-      interrupt_requested_ = true;
-    }
   }
 
   return first;

tools/replay/replay.h

@@ -85,7 +85,6 @@ private:
   std::thread stream_thread_;
   std::mutex stream_lock_;
   bool user_paused_ = false;
-  bool pause_after_next_frame_ = false;
   std::condition_variable stream_cv_;
   int current_segment_ = 0;
   std::optional<double> seeking_to_;