kinda working E2E

SConstruct

@@ -307,6 +307,7 @@ qt_flags = [
 qt_env['CXXFLAGS'] += qt_flags
 qt_env['LIBPATH'] += ['#selfdrive/ui', ]
 qt_env['LIBS'] = qt_libs
+Export('qt_libs')
 
 if GetOption("clazy"):
   checks = [

selfdrive/assets/assets.qrc

@@ -2,8 +2,11 @@
 <qresource>
   <file alias="bootstrap-icons.svg">../../third_party/bootstrap/bootstrap-icons.svg</file>
   <file>img_continue_triangle.svg</file>
+  <file>img_chffr_wheel.png</file>
   <file>img_circled_check.svg</file>
   <file>img_circled_slash.svg</file>
+  <file>img_driver_face.png</file>
+  <file>img_experimental.svg</file>
   <file>img_eye_open.svg</file>
   <file>img_eye_closed.svg</file>
   <file>icons/close.svg</file>

selfdrive/ui/SConscript

@@ -24,15 +24,17 @@ widgets_src = ["qt/widgets/input.cc", "qt/widgets/wifi.cc", "qt/prime_state.cc",
 
 widgets = qt_env.Library("qt_widgets", widgets_src, LIBS=base_libs)
 Export('widgets')
-qt_libs = [widgets, qt_util] + base_libs
+ui_libs = qt_util + widgets + base_libs
 
-qt_src = ["main.cc", "ui.cc", "qt/sidebar.cc", "qt/body.cc",
+qt_src = ["ui.cc", "qt/sidebar.cc", "qt/body.cc",
-          "qt/recorder/recorder.cc",
           "qt/window.cc", "qt/home.cc", "qt/offroad/settings.cc",
           "qt/offroad/software_settings.cc", "qt/offroad/developer_panel.cc", "qt/offroad/onboarding.cc",
           "qt/offroad/driverview.cc", "qt/offroad/experimental_mode.cc", "qt/offroad/firehose.cc",
           "qt/onroad/onroad_home.cc", "qt/onroad/annotated_camera.cc", "qt/onroad/model.cc",
           "qt/onroad/buttons.cc", "qt/onroad/alerts.cc", "qt/onroad/driver_monitoring.cc", "qt/onroad/hud.cc"]
+qt_ui = qt_env.Library("qt_ui", qt_src, LIBS=ui_libs)
+Export('qt_ui')
+Export('ui_libs')
 
 # build translation files
 with open(File("translations/languages.json").abspath) as f:
@@ -59,25 +61,26 @@ qt_env.Command(assets, [assets_src, translations_assets_src], f"rcc $SOURCES -o
 qt_env.Depends(assets, Glob('#selfdrive/assets/*', exclude=[assets, assets_src, translations_assets_src, "#selfdrive/assets/assets.o"]) + [lrelease])
 asset_obj = qt_env.Object("assets", assets)
 
+Export('asset_obj')
+
 # build main UI
-qt_env.Program("ui", qt_src + [asset_obj], LIBS=qt_libs)
+qt_env.Program("ui", ["main.cc", asset_obj] + qt_src, LIBS=ui_libs)
 if GetOption('extras'):
-  qt_src.remove("main.cc")  # replaced by test_runner
+  qt_env.Program('tests/test_translations', [asset_obj, 'tests/test_runner.cc', 'tests/test_translations.cc'] + qt_src, LIBS=ui_libs)
-  qt_env.Program('tests/test_translations', [asset_obj, 'tests/test_runner.cc', 'tests/test_translations.cc'] + qt_src, LIBS=qt_libs)
 
-  qt_env.SharedLibrary("qt/python_helpers", ["qt/qt_window.cc"], LIBS=qt_libs)
+  qt_env.SharedLibrary("qt/python_helpers", ["qt/qt_window.cc"], LIBS=ui_libs)
 
   # spinner and text window
-  qt_env.Program("_text", ["qt/text.cc"], LIBS=qt_libs)
+  qt_env.Program("_text", ["qt/text.cc"], LIBS=ui_libs)
-  qt_env.Program("_spinner", ["qt/spinner.cc"], LIBS=qt_libs)
+  qt_env.Program("_spinner", ["qt/spinner.cc"], LIBS=ui_libs)
 
   # setup and factory resetter
-  qt_env.Program("qt/setup/reset", ["qt/setup/reset.cc"], LIBS=qt_libs)
+  qt_env.Program("qt/setup/reset", ["qt/setup/reset.cc"], LIBS=ui_libs)
   qt_env.Program("qt/setup/setup", ["qt/setup/setup.cc", asset_obj],
-                 LIBS=qt_libs + ['curl', 'common'])
+                 LIBS=ui_libs + ['curl', 'common'])
 
   # build updater UI
-  qt_env.Program("qt/setup/updater", ["qt/setup/updater.cc", asset_obj], LIBS=qt_libs)
+  qt_env.Program("qt/setup/updater", ["qt/setup/updater.cc", asset_obj], LIBS=ui_libs)
 
   if arch != "Darwin":
     # build installers
@@ -100,10 +103,10 @@ if GetOption('extras'):
         d['INTERNAL'] = "1"
 
       obj = senv.Object(f"installer/installers/installer_{name}.o", ["installer/installer.cc"], CPPDEFINES=d)
-      f = senv.Program(f"installer/installers/installer_{name}", [obj, cont], LIBS=qt_libs)
+      f = senv.Program(f"installer/installers/installer_{name}", [obj, cont], LIBS=ui_libs)
       # keep installers small
       assert f[0].get_size() < 370*1e3
 
 # build watch3
 if arch in ['x86_64', 'aarch64', 'Darwin'] or GetOption('extras'):
-  qt_env.Program("watch3", ["watch3.cc"], LIBS=qt_libs + ['common', 'msgq', 'visionipc'])
+  qt_env.Program("watch3", ["watch3.cc"], LIBS=ui_libs + ['common', 'msgq', 'visionipc'])

selfdrive/ui/main.cc

@@ -20,7 +20,6 @@
 #include "selfdrive/ui/qt/qt_window.h"
 #include "selfdrive/ui/qt/util.h"
 #include "selfdrive/ui/qt/window.h"
-#include "selfdrive/ui/qt/recorder/recorder.h"
 
 int main(int argc, char *argv[]) {
   setpriority(PRIO_PROCESS, 0, -20);
@@ -63,45 +62,5 @@ int main(int argc, char *argv[]) {
   setMainWindow(&w);
   a.installEventFilter(&w);
 
-  QSurfaceFormat format;
-  format.setRenderableType(QSurfaceFormat::OpenGLES);
-  format.setVersion(3, 0);
-  format.setProfile(QSurfaceFormat::CoreProfile);
-  QSurfaceFormat::setDefaultFormat(format);
-
-  QOffscreenSurface surface;
-  surface.create();
-  QOpenGLContext context;
-  context.create();
-  context.makeCurrent(&surface);
-
-  QScopedPointer<FFmpegEncoder> encoder(new FFmpegEncoder(outputFile, DEVICE_SCREEN_SIZE.width(), DEVICE_SCREEN_SIZE.height(), 30));
-  encoder->startRecording();
-
-  QScopedPointer<QTimer> captureTimer(new QTimer);
-  QObject::connect(captureTimer.data(), &QTimer::timeout, [&]() {
-    context.makeCurrent(&surface);
-
-    QCoreApplication::processEvents();
-
-    QImage image = w.grab().toImage();
-
-    if (image.isNull() || image.size() != DEVICE_SCREEN_SIZE) {
-      qWarning() << "Invalid image captured";
-      context.doneCurrent();
-      return;
-    }
-
-    image = image.convertToFormat(QImage::Format_ARGB32_Premultiplied);
-
-    if (!encoder->writeFrame(image)) {
-      qWarning() << "Failed to write frame";
-    }
-
-    context.doneCurrent();
-  });
-
-  captureTimer->start(1000/30);
-
   return a.exec();
 }

selfdrive/ui/qt/onroad/annotated_camera.cc

@@ -131,7 +131,7 @@ void AnnotatedCameraWidget::paintGL() {
   painter.setRenderHint(QPainter::Antialiasing);
   painter.setPen(Qt::NoPen);
 
-  model.draw(painter, rect());
+  // model.draw(painter, rect());
   dmon.draw(painter, rect());
   hud.updateState(*s);
   hud.draw(painter, rect());

selfdrive/ui/qt/onroad/buttons.cc

@@ -19,8 +19,8 @@ void drawIcon(QPainter &p, const QPoint &center, const QPixmap &img, const QBrus
 ExperimentalButton::ExperimentalButton(QWidget *parent) : experimental_mode(false), engageable(false), QPushButton(parent) {
   setFixedSize(btn_size, btn_size);
 
-  engage_img = loadPixmap("../assets/img_chffr_wheel.png", {img_size, img_size});
+  engage_img = loadPixmap(":/img_chffr_wheel.png", {img_size, img_size});
-  experimental_img = loadPixmap("../assets/img_experimental.svg", {img_size, img_size});
+  experimental_img = loadPixmap(":/img_experimental.svg", {img_size, img_size});
   QObject::connect(this, &QPushButton::clicked, this, &ExperimentalButton::changeMode);
 }
 

selfdrive/ui/qt/onroad/driver_monitoring.cc

@@ -21,7 +21,7 @@ static const QColor DMON_ENGAGED_COLOR = QColor::fromRgbF(0.1, 0.945, 0.26);
 static const QColor DMON_DISENGAGED_COLOR = QColor::fromRgbF(0.545, 0.545, 0.545);
 
 DriverMonitorRenderer::DriverMonitorRenderer() : face_kpts_draw(std::size(DEFAULT_FACE_KPTS_3D)) {
-  dm_img = loadPixmap("../assets/img_driver_face.png", {img_size + 5, img_size + 5});
+  dm_img = loadPixmap(":/img_driver_face.png", {img_size + 5, img_size + 5});
 }
 
 void DriverMonitorRenderer::updateState(const UIState &s) {

selfdrive/ui/qt/onroad/onroad_home.cc

@@ -44,6 +44,8 @@ void OnroadWindow::updateState(const UIState &s) {
     return;
   }
 
+  emit drewOnroadFrame(new QImage(grab().toImage()));
+
   alerts->updateState(s);
   nvg->updateState(s);
 

selfdrive/ui/qt/onroad/onroad_home.h

@@ -9,6 +9,9 @@ class OnroadWindow : public QWidget {
 public:
   OnroadWindow(QWidget* parent = 0);
 
+signals:
+  void drewOnroadFrame(QImage *frame);
+
 private:
   void paintEvent(QPaintEvent *event);
   OnroadAlerts *alerts;

selfdrive/ui/ui.h

@@ -18,7 +18,7 @@
 const int UI_BORDER_SIZE = 30;
 const int UI_HEADER_HEIGHT = 420;
 
-const int UI_FREQ = 20; // Hz
+const int UI_FREQ = 60; // Hz
 const int BACKLIGHT_OFFROAD = 50;
 
 const Eigen::Matrix3f VIEW_FROM_DEVICE = (Eigen::Matrix3f() <<

tools/replay/SConscript

@@ -1,22 +1,25 @@
-Import('env', 'arch', 'common', 'messaging', 'visionipc', 'cereal')
+Import('qt_env', 'ui_libs', 'qt_libs', 'qt_ui', 'asset_obj', 'arch', 'common', 'messaging', 'visionipc', 'cereal')
 
-replay_env = env.Clone()
+replay_env = qt_env.Clone()
 replay_env['CCFLAGS'] += ['-Wno-deprecated-declarations']
 
-base_frameworks = []
+base_frameworks = replay_env["FRAMEWORKS"]
-base_libs = [common, messaging, cereal, visionipc, 'm', 'ssl', 'crypto', 'pthread']
+base_libs = replay_env["LIBS"]
 
 if arch == "Darwin":
   base_frameworks.append('OpenCL')
 else:
   base_libs.append('OpenCL')
 
+if arch == 'larch64':
+  base_libs.append('EGL')
+
 replay_lib_src = ["replay.cc", "consoleui.cc", "camera.cc", "filereader.cc", "logreader.cc", "framereader.cc",
-                  "route.cc", "util.cc", "seg_mgr.cc", "timeline.cc", "api.cc"]
+                  "route.cc", "util.cc", "seg_mgr.cc", "timeline.cc", "api.cc", "clip/recorder/widget.cc", "clip/recorder/ffmpeg.cc", "clip/application.cc"]
 replay_lib = replay_env.Library("replay", replay_lib_src, LIBS=base_libs, FRAMEWORKS=base_frameworks)
 Export('replay_lib')
-replay_libs = [replay_lib, 'avutil', 'avcodec', 'avformat', 'bz2', 'zstd', 'curl', 'yuv', 'ncurses'] + base_libs
+replay_libs = [replay_lib, cereal, 'bz2', 'zstd', 'curl', 'yuv', 'ncurses'] + base_libs + qt_libs + ui_libs + qt_ui
-replay_env.Program("replay", ["main.cc"], LIBS=replay_libs, FRAMEWORKS=base_frameworks)
+replay_env.Program("replay", ["main.cc", asset_obj], LIBS=replay_libs, FRAMEWORKS=base_frameworks)
 
 if GetOption('extras'):
   replay_env.Program('tests/test_replay', ['tests/test_replay.cc'], LIBS=replay_libs)

tools/replay/clip/application.cc

@@ -0,0 +1,51 @@
+#include "tools/replay/clip/application.h"
+
+#include <QApplication>
+#include <QTranslator>
+#include <selfdrive/ui/qt/util.h>
+#include <selfdrive/ui/qt/window.h>
+
+#include "recorder/widget.h"
+
+Application::Application() {
+
+}
+
+Application::~Application() {
+
+}
+
+int Application::exec(int argc, char *argv[]) {
+  initApp(argc, argv);
+
+  QApplication a(argc, argv);
+
+  QString outputFile = "/Users/trey/Desktop/out.mp4";
+
+  QTranslator translator;
+  QString translation_file = QString::fromStdString(Params().get("LanguageSetting"));
+  if (!translator.load(QString(":/%1").arg(translation_file)) && translation_file.length()) {
+    qCritical() << "Failed to load translation file:" << translation_file;
+  }
+
+  a.installTranslator(&translator);
+
+  OnroadWindow w;
+
+  QThread recorderThread;
+  Recorder recorder;
+  recorder.moveToThread(&recorderThread);
+  QObject::connect(&recorderThread, &QThread::finished, &recorder, &QObject::deleteLater);
+  QObject::connect(&w, &OnroadWindow::drewOnroadFrame, &recorder, &Recorder::saveFrame, Qt::QueuedConnection);
+  recorderThread.start();
+
+  w.setAttribute(Qt::WA_DontShowOnScreen);
+  w.setAttribute(Qt::WA_Mapped);
+  w.setAttribute(Qt::WA_NoSystemBackground);
+  w.resize(DEVICE_SCREEN_SIZE);
+  setMainWindow(&w);
+
+  return a.exec();
+}
+
+

tools/replay/clip/application.h

@@ -0,0 +1,9 @@
+#pragma once
+
+
+class Application {
+public:
+    Application();
+    ~Application();
+    int exec(int argc, char* argv[]);
+};

tools/replay/clip/recorder/ffmpeg.cc

@@ -1,4 +1,4 @@
-#include "selfdrive/ui/qt/recorder/recorder.h"
+#include "tools/replay/clip/recorder/ffmpeg.h"
 
 #include <QDebug>
 

tools/replay/clip/recorder/ffmpeg.h

@@ -6,8 +6,8 @@
 extern "C" {
 #include <libavcodec/avcodec.h>
 #include <libavformat/avformat.h>
+#include <libavutil/avutil.h>
 #include <libavutil/imgutils.h>
-#include <libavutil/dict.h>
 #include <libswscale/swscale.h>
 }
 

tools/replay/clip/recorder/widget.cc

@@ -0,0 +1,46 @@
+#include "tools/replay/clip/recorder/widget.h"
+
+#include "tools/replay/clip/recorder/ffmpeg.h"
+
+Recorder::~Recorder() {
+    fprintf(stderr, "closing\n");
+    QObject::~QObject();
+}
+
+void Recorder::saveFrame(QImage *frame) {
+    QMutexLocker locker(&mutex);
+    frameQueue.enqueue(frame); // Add frame to queue
+    if (!isProcessing) {
+        isProcessing = true;
+        QMetaObject::invokeMethod(this, &Recorder::processQueue, Qt::QueuedConnection);
+    }
+}
+
+void Recorder::processQueue() {
+    while (true) {
+        QImage *frame;
+        {
+            QMutexLocker locker(&mutex);
+            if (frameQueue.isEmpty() || !keepRunning) {
+                isProcessing = false;
+                return;
+            }
+            frame = frameQueue.dequeue();
+        }
+        // Save the frame (this runs in the worker thread)
+        static int frameCount = 0;
+        if (frameCount == 0 && !encoder.startRecording()) {
+            fprintf(stderr, "failed to start record\n");
+        }
+        fprintf(stderr, "processing frame %d: %p\n", frameCount++, &frame);
+        if (!encoder.writeFrame(frame->convertToFormat(QImage::Format_ARGB32_Premultiplied))) {
+            fprintf(stderr, "did not write\n");
+        }
+    }
+}
+
+void Recorder::stop() {
+    QMutexLocker locker(&mutex);
+    keepRunning = false;
+}
+

tools/replay/clip/recorder/widget.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <QImage>
+#include <QObject>
+#include <QQueue>
+#include <selfdrive/ui/ui.h>
+#include <selfdrive/ui/qt/qt_window.h>
+
+#include "ffmpeg.h"
+
+class Recorder : public QObject {
+    Q_OBJECT
+
+public:
+    Recorder(QObject *parent = nullptr) : QObject(parent) {}
+    ~Recorder() override;
+
+public slots:
+    void saveFrame(QImage *frame);
+    void stop();
+
+private:
+    FFmpegEncoder encoder = FFmpegEncoder("/Users/trey/Desktop/out.mp4", DEVICE_SCREEN_SIZE.width(), DEVICE_SCREEN_SIZE.height(), UI_FREQ);
+    QQueue<QImage *> frameQueue;
+    QMutex mutex;
+    bool isProcessing = false;
+    bool keepRunning = true;
+    void processQueue();
+};

tools/replay/main.cc

@@ -5,6 +5,7 @@
 #include <string>
 #include <vector>
 
+#include "clip/application.h"
 #include "common/prefix.h"
 #include "tools/replay/consoleui.h"
 #include "tools/replay/replay.h"
@@ -40,6 +41,7 @@ struct ReplayConfig {
   std::string prefix;
   uint32_t flags = REPLAY_FLAG_NONE;
   int start_seconds = 0;
+  int end_seconds = 0;
   int cache_segments = -1;
   float playback_speed = -1;
 };
@@ -50,6 +52,7 @@ bool parseArgs(int argc, char *argv[], ReplayConfig &config) {
       {"block", required_argument, nullptr, 'b'},
       {"cache", required_argument, nullptr, 'c'},
       {"start", required_argument, nullptr, 's'},
+      {"end", required_argument, nullptr, 'e'},
       {"playback", required_argument, nullptr, 'x'},
       {"demo", no_argument, nullptr, 0},
       {"data_dir", required_argument, nullptr, 'd'},
@@ -62,6 +65,7 @@ bool parseArgs(int argc, char *argv[], ReplayConfig &config) {
       {"no-hw-decoder", no_argument, nullptr, 0},
       {"no-vipc", no_argument, nullptr, 0},
       {"all", no_argument, nullptr, 0},
+      {"clip", no_argument, nullptr, 0},
       {"help", no_argument, nullptr, 'h'},
       {nullptr, 0, nullptr, 0},  // Terminating entry
   };
@@ -75,6 +79,7 @@ bool parseArgs(int argc, char *argv[], ReplayConfig &config) {
       {"no-hw-decoder", REPLAY_FLAG_NO_HW_DECODER},
       {"no-vipc", REPLAY_FLAG_NO_VIPC},
       {"all", REPLAY_FLAG_ALL_SERVICES},
+      {"clip", REPLAY_FLAG_CLIP},
   };
 
   if (argc == 1) {
@@ -88,6 +93,7 @@ bool parseArgs(int argc, char *argv[], ReplayConfig &config) {
       case 'a': config.allow = split(optarg, ','); break;
       case 'b': config.block = split(optarg, ','); break;
       case 'c': config.cache_segments = std::atoi(optarg); break;
+      case 'e': config.end_seconds = std::atoi(optarg); break;
       case 's': config.start_seconds = std::atoi(optarg); break;
       case 'x': config.playback_speed = std::atof(optarg); break;
       case 'd': config.data_dir = optarg; break;
@@ -147,7 +153,14 @@ int main(int argc, char *argv[]) {
     return 1;
   }
 
-  ConsoleUI console_ui(&replay);
   replay.start(config.start_seconds);
-  return console_ui.exec();
+
+  if (replay.hasFlag(REPLAY_FLAG_CLIP)) {
+    Application a;
+    // kick off
+    return a.exec(argc, argv);
+  } else {
+    ConsoleUI console_ui(&replay);
+    return console_ui.exec();
+  }
 }

tools/replay/replay.h

@@ -24,6 +24,7 @@ enum REPLAY_FLAGS {
   REPLAY_FLAG_NO_HW_DECODER = 0x0100,
   REPLAY_FLAG_NO_VIPC = 0x0400,
   REPLAY_FLAG_ALL_SERVICES = 0x0800,
+  REPLAY_FLAG_CLIP = 0x1000,
 };
 
 class Replay {