diff --git a/SConstruct b/SConstruct index c40f118985..e99fd01f9a 100644 --- a/SConstruct +++ b/SConstruct @@ -307,6 +307,7 @@ qt_flags = [ qt_env['CXXFLAGS'] += qt_flags qt_env['LIBPATH'] += ['#selfdrive/ui', ] qt_env['LIBS'] = qt_libs +Export('qt_libs') if GetOption("clazy"): checks = [ diff --git a/selfdrive/assets/assets.qrc b/selfdrive/assets/assets.qrc index 6d8d8df334..d27d9baf38 100644 --- a/selfdrive/assets/assets.qrc +++ b/selfdrive/assets/assets.qrc @@ -2,8 +2,11 @@ ../../third_party/bootstrap/bootstrap-icons.svg img_continue_triangle.svg + img_chffr_wheel.png img_circled_check.svg img_circled_slash.svg + img_driver_face.png + img_experimental.svg img_eye_open.svg img_eye_closed.svg icons/close.svg diff --git a/selfdrive/ui/SConscript b/selfdrive/ui/SConscript index e5a1742bae..ce786e35dd 100644 --- a/selfdrive/ui/SConscript +++ b/selfdrive/ui/SConscript @@ -24,15 +24,17 @@ widgets_src = ["qt/widgets/input.cc", "qt/widgets/wifi.cc", "qt/prime_state.cc", widgets = qt_env.Library("qt_widgets", widgets_src, LIBS=base_libs) Export('widgets') -qt_libs = [widgets, qt_util] + base_libs +ui_libs = qt_util + widgets + base_libs -qt_src = ["main.cc", "ui.cc", "qt/sidebar.cc", "qt/body.cc", - "qt/recorder/recorder.cc", +qt_src = ["ui.cc", "qt/sidebar.cc", "qt/body.cc", "qt/window.cc", "qt/home.cc", "qt/offroad/settings.cc", "qt/offroad/software_settings.cc", "qt/offroad/developer_panel.cc", "qt/offroad/onboarding.cc", "qt/offroad/driverview.cc", "qt/offroad/experimental_mode.cc", "qt/offroad/firehose.cc", "qt/onroad/onroad_home.cc", "qt/onroad/annotated_camera.cc", "qt/onroad/model.cc", "qt/onroad/buttons.cc", "qt/onroad/alerts.cc", "qt/onroad/driver_monitoring.cc", "qt/onroad/hud.cc"] +qt_ui = qt_env.Library("qt_ui", qt_src, LIBS=ui_libs) +Export('qt_ui') +Export('ui_libs') # build translation files with open(File("translations/languages.json").abspath) as f: @@ -59,25 +61,26 @@ qt_env.Command(assets, [assets_src, translations_assets_src], f"rcc $SOURCES -o qt_env.Depends(assets, Glob('#selfdrive/assets/*', exclude=[assets, assets_src, translations_assets_src, "#selfdrive/assets/assets.o"]) + [lrelease]) asset_obj = qt_env.Object("assets", assets) +Export('asset_obj') + # build main UI -qt_env.Program("ui", qt_src + [asset_obj], LIBS=qt_libs) +qt_env.Program("ui", ["main.cc", asset_obj] + qt_src, LIBS=ui_libs) if GetOption('extras'): - qt_src.remove("main.cc") # replaced by test_runner - qt_env.Program('tests/test_translations', [asset_obj, 'tests/test_runner.cc', 'tests/test_translations.cc'] + qt_src, LIBS=qt_libs) + qt_env.Program('tests/test_translations', [asset_obj, 'tests/test_runner.cc', 'tests/test_translations.cc'] + qt_src, LIBS=ui_libs) - qt_env.SharedLibrary("qt/python_helpers", ["qt/qt_window.cc"], LIBS=qt_libs) + qt_env.SharedLibrary("qt/python_helpers", ["qt/qt_window.cc"], LIBS=ui_libs) # spinner and text window - qt_env.Program("_text", ["qt/text.cc"], LIBS=qt_libs) - qt_env.Program("_spinner", ["qt/spinner.cc"], LIBS=qt_libs) + qt_env.Program("_text", ["qt/text.cc"], LIBS=ui_libs) + qt_env.Program("_spinner", ["qt/spinner.cc"], LIBS=ui_libs) # setup and factory resetter - qt_env.Program("qt/setup/reset", ["qt/setup/reset.cc"], LIBS=qt_libs) + qt_env.Program("qt/setup/reset", ["qt/setup/reset.cc"], LIBS=ui_libs) qt_env.Program("qt/setup/setup", ["qt/setup/setup.cc", asset_obj], - LIBS=qt_libs + ['curl', 'common']) + LIBS=ui_libs + ['curl', 'common']) # build updater UI - qt_env.Program("qt/setup/updater", ["qt/setup/updater.cc", asset_obj], LIBS=qt_libs) + qt_env.Program("qt/setup/updater", ["qt/setup/updater.cc", asset_obj], LIBS=ui_libs) if arch != "Darwin": # build installers @@ -100,10 +103,10 @@ if GetOption('extras'): d['INTERNAL'] = "1" obj = senv.Object(f"installer/installers/installer_{name}.o", ["installer/installer.cc"], CPPDEFINES=d) - f = senv.Program(f"installer/installers/installer_{name}", [obj, cont], LIBS=qt_libs) + f = senv.Program(f"installer/installers/installer_{name}", [obj, cont], LIBS=ui_libs) # keep installers small assert f[0].get_size() < 370*1e3 # build watch3 if arch in ['x86_64', 'aarch64', 'Darwin'] or GetOption('extras'): - qt_env.Program("watch3", ["watch3.cc"], LIBS=qt_libs + ['common', 'msgq', 'visionipc']) + qt_env.Program("watch3", ["watch3.cc"], LIBS=ui_libs + ['common', 'msgq', 'visionipc']) diff --git a/selfdrive/ui/main.cc b/selfdrive/ui/main.cc index 0d1e1f6584..19b15d3869 100644 --- a/selfdrive/ui/main.cc +++ b/selfdrive/ui/main.cc @@ -20,7 +20,6 @@ #include "selfdrive/ui/qt/qt_window.h" #include "selfdrive/ui/qt/util.h" #include "selfdrive/ui/qt/window.h" -#include "selfdrive/ui/qt/recorder/recorder.h" int main(int argc, char *argv[]) { setpriority(PRIO_PROCESS, 0, -20); @@ -63,45 +62,5 @@ int main(int argc, char *argv[]) { setMainWindow(&w); a.installEventFilter(&w); - QSurfaceFormat format; - format.setRenderableType(QSurfaceFormat::OpenGLES); - format.setVersion(3, 0); - format.setProfile(QSurfaceFormat::CoreProfile); - QSurfaceFormat::setDefaultFormat(format); - - QOffscreenSurface surface; - surface.create(); - QOpenGLContext context; - context.create(); - context.makeCurrent(&surface); - - QScopedPointer encoder(new FFmpegEncoder(outputFile, DEVICE_SCREEN_SIZE.width(), DEVICE_SCREEN_SIZE.height(), 30)); - encoder->startRecording(); - - QScopedPointer captureTimer(new QTimer); - QObject::connect(captureTimer.data(), &QTimer::timeout, [&]() { - context.makeCurrent(&surface); - - QCoreApplication::processEvents(); - - QImage image = w.grab().toImage(); - - if (image.isNull() || image.size() != DEVICE_SCREEN_SIZE) { - qWarning() << "Invalid image captured"; - context.doneCurrent(); - return; - } - - image = image.convertToFormat(QImage::Format_ARGB32_Premultiplied); - - if (!encoder->writeFrame(image)) { - qWarning() << "Failed to write frame"; - } - - context.doneCurrent(); - }); - - captureTimer->start(1000/30); - return a.exec(); } diff --git a/selfdrive/ui/qt/onroad/annotated_camera.cc b/selfdrive/ui/qt/onroad/annotated_camera.cc index aed6a3c41a..fcf645a493 100644 --- a/selfdrive/ui/qt/onroad/annotated_camera.cc +++ b/selfdrive/ui/qt/onroad/annotated_camera.cc @@ -131,7 +131,7 @@ void AnnotatedCameraWidget::paintGL() { painter.setRenderHint(QPainter::Antialiasing); painter.setPen(Qt::NoPen); - model.draw(painter, rect()); + // model.draw(painter, rect()); dmon.draw(painter, rect()); hud.updateState(*s); hud.draw(painter, rect()); diff --git a/selfdrive/ui/qt/onroad/buttons.cc b/selfdrive/ui/qt/onroad/buttons.cc index 2c2cc672b9..f49b1e2d65 100644 --- a/selfdrive/ui/qt/onroad/buttons.cc +++ b/selfdrive/ui/qt/onroad/buttons.cc @@ -19,8 +19,8 @@ void drawIcon(QPainter &p, const QPoint ¢er, const QPixmap &img, const QBrus ExperimentalButton::ExperimentalButton(QWidget *parent) : experimental_mode(false), engageable(false), QPushButton(parent) { setFixedSize(btn_size, btn_size); - engage_img = loadPixmap("../assets/img_chffr_wheel.png", {img_size, img_size}); - experimental_img = loadPixmap("../assets/img_experimental.svg", {img_size, img_size}); + engage_img = loadPixmap(":/img_chffr_wheel.png", {img_size, img_size}); + experimental_img = loadPixmap(":/img_experimental.svg", {img_size, img_size}); QObject::connect(this, &QPushButton::clicked, this, &ExperimentalButton::changeMode); } diff --git a/selfdrive/ui/qt/onroad/driver_monitoring.cc b/selfdrive/ui/qt/onroad/driver_monitoring.cc index afd003cf8f..e6f484bc3a 100644 --- a/selfdrive/ui/qt/onroad/driver_monitoring.cc +++ b/selfdrive/ui/qt/onroad/driver_monitoring.cc @@ -21,7 +21,7 @@ static const QColor DMON_ENGAGED_COLOR = QColor::fromRgbF(0.1, 0.945, 0.26); static const QColor DMON_DISENGAGED_COLOR = QColor::fromRgbF(0.545, 0.545, 0.545); DriverMonitorRenderer::DriverMonitorRenderer() : face_kpts_draw(std::size(DEFAULT_FACE_KPTS_3D)) { - dm_img = loadPixmap("../assets/img_driver_face.png", {img_size + 5, img_size + 5}); + dm_img = loadPixmap(":/img_driver_face.png", {img_size + 5, img_size + 5}); } void DriverMonitorRenderer::updateState(const UIState &s) { diff --git a/selfdrive/ui/qt/onroad/onroad_home.cc b/selfdrive/ui/qt/onroad/onroad_home.cc index 080f9bd50f..a627df9928 100644 --- a/selfdrive/ui/qt/onroad/onroad_home.cc +++ b/selfdrive/ui/qt/onroad/onroad_home.cc @@ -44,6 +44,8 @@ void OnroadWindow::updateState(const UIState &s) { return; } + emit drewOnroadFrame(new QImage(grab().toImage())); + alerts->updateState(s); nvg->updateState(s); diff --git a/selfdrive/ui/qt/onroad/onroad_home.h b/selfdrive/ui/qt/onroad/onroad_home.h index c321d2d44f..46c32e33a7 100644 --- a/selfdrive/ui/qt/onroad/onroad_home.h +++ b/selfdrive/ui/qt/onroad/onroad_home.h @@ -9,6 +9,9 @@ class OnroadWindow : public QWidget { public: OnroadWindow(QWidget* parent = 0); +signals: + void drewOnroadFrame(QImage *frame); + private: void paintEvent(QPaintEvent *event); OnroadAlerts *alerts; diff --git a/selfdrive/ui/ui.h b/selfdrive/ui/ui.h index 6ff67cacde..03b52044b7 100644 --- a/selfdrive/ui/ui.h +++ b/selfdrive/ui/ui.h @@ -18,7 +18,7 @@ const int UI_BORDER_SIZE = 30; const int UI_HEADER_HEIGHT = 420; -const int UI_FREQ = 20; // Hz +const int UI_FREQ = 60; // Hz const int BACKLIGHT_OFFROAD = 50; const Eigen::Matrix3f VIEW_FROM_DEVICE = (Eigen::Matrix3f() << diff --git a/tools/replay/SConscript b/tools/replay/SConscript index 18849407cf..f61d39ed45 100644 --- a/tools/replay/SConscript +++ b/tools/replay/SConscript @@ -1,22 +1,25 @@ -Import('env', 'arch', 'common', 'messaging', 'visionipc', 'cereal') +Import('qt_env', 'ui_libs', 'qt_libs', 'qt_ui', 'asset_obj', 'arch', 'common', 'messaging', 'visionipc', 'cereal') -replay_env = env.Clone() +replay_env = qt_env.Clone() replay_env['CCFLAGS'] += ['-Wno-deprecated-declarations'] -base_frameworks = [] -base_libs = [common, messaging, cereal, visionipc, 'm', 'ssl', 'crypto', 'pthread'] +base_frameworks = replay_env["FRAMEWORKS"] +base_libs = replay_env["LIBS"] if arch == "Darwin": base_frameworks.append('OpenCL') else: base_libs.append('OpenCL') +if arch == 'larch64': + base_libs.append('EGL') + replay_lib_src = ["replay.cc", "consoleui.cc", "camera.cc", "filereader.cc", "logreader.cc", "framereader.cc", - "route.cc", "util.cc", "seg_mgr.cc", "timeline.cc", "api.cc"] + "route.cc", "util.cc", "seg_mgr.cc", "timeline.cc", "api.cc", "clip/recorder/widget.cc", "clip/recorder/ffmpeg.cc", "clip/application.cc"] replay_lib = replay_env.Library("replay", replay_lib_src, LIBS=base_libs, FRAMEWORKS=base_frameworks) Export('replay_lib') -replay_libs = [replay_lib, 'avutil', 'avcodec', 'avformat', 'bz2', 'zstd', 'curl', 'yuv', 'ncurses'] + base_libs -replay_env.Program("replay", ["main.cc"], LIBS=replay_libs, FRAMEWORKS=base_frameworks) +replay_libs = [replay_lib, cereal, 'bz2', 'zstd', 'curl', 'yuv', 'ncurses'] + base_libs + qt_libs + ui_libs + qt_ui +replay_env.Program("replay", ["main.cc", asset_obj], LIBS=replay_libs, FRAMEWORKS=base_frameworks) if GetOption('extras'): replay_env.Program('tests/test_replay', ['tests/test_replay.cc'], LIBS=replay_libs) diff --git a/tools/replay/clip/application.cc b/tools/replay/clip/application.cc new file mode 100644 index 0000000000..1bdb34aeba --- /dev/null +++ b/tools/replay/clip/application.cc @@ -0,0 +1,51 @@ +#include "tools/replay/clip/application.h" + +#include +#include +#include +#include + +#include "recorder/widget.h" + +Application::Application() { + +} + +Application::~Application() { + +} + +int Application::exec(int argc, char *argv[]) { + initApp(argc, argv); + + QApplication a(argc, argv); + + QString outputFile = "/Users/trey/Desktop/out.mp4"; + + QTranslator translator; + QString translation_file = QString::fromStdString(Params().get("LanguageSetting")); + if (!translator.load(QString(":/%1").arg(translation_file)) && translation_file.length()) { + qCritical() << "Failed to load translation file:" << translation_file; + } + + a.installTranslator(&translator); + + OnroadWindow w; + + QThread recorderThread; + Recorder recorder; + recorder.moveToThread(&recorderThread); + QObject::connect(&recorderThread, &QThread::finished, &recorder, &QObject::deleteLater); + QObject::connect(&w, &OnroadWindow::drewOnroadFrame, &recorder, &Recorder::saveFrame, Qt::QueuedConnection); + recorderThread.start(); + + w.setAttribute(Qt::WA_DontShowOnScreen); + w.setAttribute(Qt::WA_Mapped); + w.setAttribute(Qt::WA_NoSystemBackground); + w.resize(DEVICE_SCREEN_SIZE); + setMainWindow(&w); + + return a.exec(); +} + + diff --git a/tools/replay/clip/application.h b/tools/replay/clip/application.h new file mode 100644 index 0000000000..159d0ea687 --- /dev/null +++ b/tools/replay/clip/application.h @@ -0,0 +1,9 @@ +#pragma once + + +class Application { +public: + Application(); + ~Application(); + int exec(int argc, char* argv[]); +}; \ No newline at end of file diff --git a/selfdrive/ui/qt/recorder/recorder.cc b/tools/replay/clip/recorder/ffmpeg.cc similarity index 98% rename from selfdrive/ui/qt/recorder/recorder.cc rename to tools/replay/clip/recorder/ffmpeg.cc index 24b3ef8d09..194c8a54f8 100644 --- a/selfdrive/ui/qt/recorder/recorder.cc +++ b/tools/replay/clip/recorder/ffmpeg.cc @@ -1,4 +1,4 @@ -#include "selfdrive/ui/qt/recorder/recorder.h" +#include "tools/replay/clip/recorder/ffmpeg.h" #include diff --git a/selfdrive/ui/qt/recorder/recorder.h b/tools/replay/clip/recorder/ffmpeg.h similarity index 95% rename from selfdrive/ui/qt/recorder/recorder.h rename to tools/replay/clip/recorder/ffmpeg.h index 669044efb5..8a608739bc 100644 --- a/selfdrive/ui/qt/recorder/recorder.h +++ b/tools/replay/clip/recorder/ffmpeg.h @@ -6,8 +6,8 @@ extern "C" { #include #include +#include #include -#include #include } diff --git a/tools/replay/clip/recorder/widget.cc b/tools/replay/clip/recorder/widget.cc new file mode 100644 index 0000000000..f701f5015f --- /dev/null +++ b/tools/replay/clip/recorder/widget.cc @@ -0,0 +1,46 @@ +#include "tools/replay/clip/recorder/widget.h" + +#include "tools/replay/clip/recorder/ffmpeg.h" + +Recorder::~Recorder() { + fprintf(stderr, "closing\n"); + QObject::~QObject(); +} + +void Recorder::saveFrame(QImage *frame) { + QMutexLocker locker(&mutex); + frameQueue.enqueue(frame); // Add frame to queue + if (!isProcessing) { + isProcessing = true; + QMetaObject::invokeMethod(this, &Recorder::processQueue, Qt::QueuedConnection); + } +} + +void Recorder::processQueue() { + while (true) { + QImage *frame; + { + QMutexLocker locker(&mutex); + if (frameQueue.isEmpty() || !keepRunning) { + isProcessing = false; + return; + } + frame = frameQueue.dequeue(); + } + // Save the frame (this runs in the worker thread) + static int frameCount = 0; + if (frameCount == 0 && !encoder.startRecording()) { + fprintf(stderr, "failed to start record\n"); + } + fprintf(stderr, "processing frame %d: %p\n", frameCount++, &frame); + if (!encoder.writeFrame(frame->convertToFormat(QImage::Format_ARGB32_Premultiplied))) { + fprintf(stderr, "did not write\n"); + } + } +} + +void Recorder::stop() { + QMutexLocker locker(&mutex); + keepRunning = false; +} + diff --git a/tools/replay/clip/recorder/widget.h b/tools/replay/clip/recorder/widget.h new file mode 100644 index 0000000000..e976a990d6 --- /dev/null +++ b/tools/replay/clip/recorder/widget.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "ffmpeg.h" + +class Recorder : public QObject { + Q_OBJECT + +public: + Recorder(QObject *parent = nullptr) : QObject(parent) {} + ~Recorder() override; + +public slots: + void saveFrame(QImage *frame); + void stop(); + +private: + FFmpegEncoder encoder = FFmpegEncoder("/Users/trey/Desktop/out.mp4", DEVICE_SCREEN_SIZE.width(), DEVICE_SCREEN_SIZE.height(), UI_FREQ); + QQueue frameQueue; + QMutex mutex; + bool isProcessing = false; + bool keepRunning = true; + void processQueue(); +}; diff --git a/tools/replay/main.cc b/tools/replay/main.cc index b33b7fa263..8904e4730d 100644 --- a/tools/replay/main.cc +++ b/tools/replay/main.cc @@ -5,6 +5,7 @@ #include #include +#include "clip/application.h" #include "common/prefix.h" #include "tools/replay/consoleui.h" #include "tools/replay/replay.h" @@ -40,6 +41,7 @@ struct ReplayConfig { std::string prefix; uint32_t flags = REPLAY_FLAG_NONE; int start_seconds = 0; + int end_seconds = 0; int cache_segments = -1; float playback_speed = -1; }; @@ -50,6 +52,7 @@ bool parseArgs(int argc, char *argv[], ReplayConfig &config) { {"block", required_argument, nullptr, 'b'}, {"cache", required_argument, nullptr, 'c'}, {"start", required_argument, nullptr, 's'}, + {"end", required_argument, nullptr, 'e'}, {"playback", required_argument, nullptr, 'x'}, {"demo", no_argument, nullptr, 0}, {"data_dir", required_argument, nullptr, 'd'}, @@ -62,6 +65,7 @@ bool parseArgs(int argc, char *argv[], ReplayConfig &config) { {"no-hw-decoder", no_argument, nullptr, 0}, {"no-vipc", no_argument, nullptr, 0}, {"all", no_argument, nullptr, 0}, + {"clip", no_argument, nullptr, 0}, {"help", no_argument, nullptr, 'h'}, {nullptr, 0, nullptr, 0}, // Terminating entry }; @@ -75,6 +79,7 @@ bool parseArgs(int argc, char *argv[], ReplayConfig &config) { {"no-hw-decoder", REPLAY_FLAG_NO_HW_DECODER}, {"no-vipc", REPLAY_FLAG_NO_VIPC}, {"all", REPLAY_FLAG_ALL_SERVICES}, + {"clip", REPLAY_FLAG_CLIP}, }; if (argc == 1) { @@ -88,6 +93,7 @@ bool parseArgs(int argc, char *argv[], ReplayConfig &config) { case 'a': config.allow = split(optarg, ','); break; case 'b': config.block = split(optarg, ','); break; case 'c': config.cache_segments = std::atoi(optarg); break; + case 'e': config.end_seconds = std::atoi(optarg); break; case 's': config.start_seconds = std::atoi(optarg); break; case 'x': config.playback_speed = std::atof(optarg); break; case 'd': config.data_dir = optarg; break; @@ -147,7 +153,14 @@ int main(int argc, char *argv[]) { return 1; } - ConsoleUI console_ui(&replay); replay.start(config.start_seconds); - return console_ui.exec(); + + if (replay.hasFlag(REPLAY_FLAG_CLIP)) { + Application a; + // kick off + return a.exec(argc, argv); + } else { + ConsoleUI console_ui(&replay); + return console_ui.exec(); + } } diff --git a/tools/replay/replay.h b/tools/replay/replay.h index 6a2c86ff02..fdf1bd4790 100644 --- a/tools/replay/replay.h +++ b/tools/replay/replay.h @@ -24,6 +24,7 @@ enum REPLAY_FLAGS { REPLAY_FLAG_NO_HW_DECODER = 0x0100, REPLAY_FLAG_NO_VIPC = 0x0400, REPLAY_FLAG_ALL_SERVICES = 0x0800, + REPLAY_FLAG_CLIP = 0x1000, }; class Replay {