Merge 2d37a29980 into 37e86df41e

tools/cabana/README.md

@@ -12,6 +12,8 @@ Options:
   -h, --help                     Displays help on commandline options.
   --help-all                     Displays help including Qt specific options.
   --demo                         use a demo route instead of providing your own
+  --auto                         Auto load the route from the best available source (no video):
+                                 internal, openpilotci, comma_api, car_segments, testing_closet
   --qcam                         load qcamera
   --ecam                         load wide road camera
   --msgq                         read can messages from msgq

tools/cabana/cabana.cc

@@ -23,6 +23,7 @@ int main(int argc, char *argv[]) {
   cmd_parser.addHelpOption();
   cmd_parser.addPositionalArgument("route", "the drive to replay. find your drives at connect.comma.ai");
   cmd_parser.addOption({"demo", "use a demo route instead of providing your own"});
+  cmd_parser.addOption({"auto", "Auto load the route from the best available source (no video): internal, openpilotci, comma_api, car_segments, testing_closet"});
   cmd_parser.addOption({"qcam", "load qcamera"});
   cmd_parser.addOption({"ecam", "load wide road camera"});
   cmd_parser.addOption({"dcam", "load driver camera"});
@@ -69,7 +70,8 @@ int main(int argc, char *argv[]) {
     }
     if (!route.isEmpty()) {
       auto replay_stream = std::make_unique<ReplayStream>(&app);
-      if (!replay_stream->loadRoute(route, cmd_parser.value("data_dir"), replay_flags)) {
+      bool auto_source = cmd_parser.isSet("auto");
+      if (!replay_stream->loadRoute(route, cmd_parser.value("data_dir"), replay_flags, auto_source)) {
         return 0;
       }
       stream = replay_stream.release();

tools/cabana/streams/replaystream.cc

@@ -7,6 +7,7 @@
 #include <QPushButton>
 
 #include "common/timing.h"
+#include "common/util.h"
 #include "tools/cabana/streams/routes.h"
 
 ReplayStream::ReplayStream(QObject *parent) : AbstractStream(parent) {
@@ -45,9 +46,9 @@ void ReplayStream::mergeSegments() {
   }
 }
 
-bool ReplayStream::loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags) {
+bool ReplayStream::loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags, bool auto_source) {
   replay.reset(new Replay(route.toStdString(), {"can", "roadEncodeIdx", "driverEncodeIdx", "wideRoadEncodeIdx", "carParams"},
-                          {}, nullptr, replay_flags, data_dir.toStdString()));
+                          {}, nullptr, replay_flags, data_dir.toStdString(), auto_source));
   replay->setSegmentCacheLimit(settings.max_cached_minutes);
   replay->installEventFilter([this](const Event *event) { return eventFilter(event); });
 

tools/cabana/streams/replaystream.h

@@ -18,7 +18,7 @@ class ReplayStream : public AbstractStream {
 public:
   ReplayStream(QObject *parent);
   void start() override { replay->start(); }
-  bool loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags = REPLAY_FLAG_NONE);
+  bool loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags = REPLAY_FLAG_NONE, bool auto_source = false);
   bool eventFilter(const Event *event);
   void seekTo(double ts) override { replay->seekTo(std::max(double(0), ts), false); }
   bool liveStreaming() const override { return false; }

tools/lib/logreader.py

@@ -6,12 +6,12 @@ import capnp
 import enum
 import os
 import pathlib
-import sys
 import tqdm
 import urllib.parse
 import warnings
 import zstandard as zstd
 
+from argparse import ArgumentParser
 from collections.abc import Callable, Iterable, Iterator
 from urllib.parse import parse_qs, urlparse
 
@@ -333,7 +333,15 @@ if __name__ == "__main__":
   # capnproto <= 0.8.0 throws errors converting byte data to string
   # below line catches those errors and replaces the bytes with \x__
   codecs.register_error("strict", codecs.backslashreplace_errors)
-  log_path = sys.argv[1]
+
-  lr = LogReader(log_path, sort_by_time=True)
+  parser = ArgumentParser(description="Process a log file and print identifiers or full messages.")
-  for msg in lr:
+  parser.add_argument("log_path", help="Path to the log file")
-    print(msg)
+  parser.add_argument("--identifiers-only", action="store_true", help="Print only log identifiers")
+  args = parser.parse_args()
+
+  lr = LogReader(args.log_path, sort_by_time=True)
+  if args.identifiers_only:
+    print("\n".join(lr.logreader_identifiers))
+  else:
+    for msg in lr:
+      print(msg)

tools/replay/README.md

@@ -64,6 +64,8 @@ Options:
   -s, --start <seconds>  start from <seconds>
   -x <speed>             playback <speed>. between 0.2 - 3
   --demo                 use a demo route instead of providing your own
+  --auto                 Auto load the route from the best available source (no video):
+                         internal, openpilotci, comma_api, car_segments, testing_closet
   --data_dir <data_dir>  local directory with routes
   --prefix <prefix>      set OPENPILOT_PREFIX
   --dcam                 load driver camera

tools/replay/main.cc

@@ -19,6 +19,8 @@ Options:
   -s, --start        Start from <seconds>
   -x, --playback     Playback <speed>
       --demo         Use a demo route instead of providing your own
+      --auto         Auto load the route from the best available source (no video):
+                     internal, openpilotci, comma_api, car_segments, testing_closet
   -d, --data_dir     Local directory with routes
   -p, --prefix       Set OPENPILOT_PREFIX
       --dcam         Load driver camera
@@ -39,6 +41,7 @@ struct ReplayConfig {
   std::string data_dir;
   std::string prefix;
   uint32_t flags = REPLAY_FLAG_NONE;
+  bool auto_source = false;
   int start_seconds = 0;
   int cache_segments = -1;
   float playback_speed = -1;
@@ -52,6 +55,7 @@ bool parseArgs(int argc, char *argv[], ReplayConfig &config) {
       {"start", required_argument, nullptr, 's'},
       {"playback", required_argument, nullptr, 'x'},
       {"demo", no_argument, nullptr, 0},
+      {"auto", no_argument, nullptr, 0},
       {"data_dir", required_argument, nullptr, 'd'},
       {"prefix", required_argument, nullptr, 'p'},
       {"dcam", no_argument, nullptr, 0},
@@ -94,11 +98,9 @@ bool parseArgs(int argc, char *argv[], ReplayConfig &config) {
       case 'p': config.prefix = optarg; break;
       case 0: {
         std::string name = cli_options[option_index].name;
-        if (name == "demo") {
+        if (name == "demo") config.route = DEMO_ROUTE;
-          config.route = DEMO_ROUTE;
+        else if (name == "auto") config.auto_source = true;
-        } else {
+        else config.flags |= flag_map.at(name);
-          config.flags |= flag_map.at(name);
-        }
         break;
       }
       case 'h': std::cout << helpText; return false;
@@ -136,7 +138,7 @@ int main(int argc, char *argv[]) {
     op_prefix = std::make_unique<OpenpilotPrefix>(config.prefix);
   }
 
-  Replay replay(config.route, config.allow, config.block, nullptr, config.flags, config.data_dir);
+  Replay replay(config.route, config.allow, config.block, nullptr, config.flags, config.data_dir, config.auto_source);
   if (config.cache_segments > 0) {
     replay.setSegmentCacheLimit(config.cache_segments);
   }

tools/replay/replay.cc

@@ -15,8 +15,8 @@ void notifyEvent(Callback &callback, Args &&...args) {
 }
 
 Replay::Replay(const std::string &route, std::vector<std::string> allow, std::vector<std::string> block,
-               SubMaster *sm, uint32_t flags, const std::string &data_dir)
+               SubMaster *sm, uint32_t flags, const std::string &data_dir, bool auto_source)
-    : sm_(sm), flags_(flags), seg_mgr_(std::make_unique<SegmentManager>(route, flags, data_dir)) {
+    : sm_(sm), flags_(flags), seg_mgr_(std::make_unique<SegmentManager>(route, flags, data_dir, auto_source)) {
   std::signal(SIGUSR1, interrupt_sleep_handler);
 
   if (!(flags_ & REPLAY_FLAG_ALL_SERVICES)) {

tools/replay/replay.h

@@ -29,7 +29,7 @@ enum REPLAY_FLAGS {
 class Replay {
 public:
   Replay(const std::string &route, std::vector<std::string> allow, std::vector<std::string> block, SubMaster *sm = nullptr,
-         uint32_t flags = REPLAY_FLAG_NONE, const std::string &data_dir = "");
+         uint32_t flags = REPLAY_FLAG_NONE, const std::string &data_dir = "", bool auto_source = false);
   ~Replay();
   bool load();
   RouteLoadError lastRouteError() const { return route().lastError(); }

tools/replay/route.cc

@@ -10,9 +10,8 @@
 #include "tools/replay/replay.h"
 #include "tools/replay/util.h"
 
-Route::Route(const std::string &route, const std::string &data_dir) : data_dir_(data_dir) {
+Route::Route(const std::string &route, const std::string &data_dir, bool auto_source)
-  route_ = parseRoute(route);
+    : route_string_(route), data_dir_(data_dir), auto_source_(auto_source) {}
-}
 
 RouteIdentifier Route::parseRoute(const std::string &str) {
   RouteIdentifier identifier = {};
@@ -22,6 +21,7 @@ RouteIdentifier Route::parseRoute(const std::string &str) {
   if (std::regex_match(str, match, pattern)) {
     identifier.dongle_id = match[2].str();
     identifier.timestamp = match[3].str();
+    identifier.date_time = strToTime(identifier.timestamp);
     identifier.str = identifier.dongle_id + "|" + identifier.timestamp;
 
     const auto separator = match[5].str();
@@ -44,27 +44,50 @@ RouteIdentifier Route::parseRoute(const std::string &str) {
 }
 
 bool Route::load() {
-  err_ = RouteLoadError::None;
+  route_ = parseRoute(route_string_);
   if (route_.str.empty() || (data_dir_.empty() && route_.dongle_id.empty())) {
     rInfo("invalid route format");
     return false;
   }
 
-  struct tm tm_time = {0};
+  if (!loadSegments()) {
-  strptime(route_.timestamp.c_str(), "%Y-%m-%d--%H-%M-%S", &tm_time);
+    rInfo("Failed to load segments");
-  date_time_ = mktime(&tm_time);
+    return false;
-
+  }
-  bool ret = data_dir_.empty() ? loadFromServer() : loadFromLocal();
+
-  if (ret) {
+  return true;
-    if (route_.begin_segment == -1) route_.begin_segment = segments_.rbegin()->first;
+}
-    if (route_.end_segment == -1) route_.end_segment = segments_.rbegin()->first;
+
-    for (auto it = segments_.begin(); it != segments_.end(); /**/) {
+bool Route::loadSegments() {
-      if (it->first < route_.begin_segment || it->first > route_.end_segment) {
+  if (!auto_source_) {
-        it = segments_.erase(it);
+    bool ret = data_dir_.empty() ? loadFromServer() : loadFromLocal();
-      } else {
+    if (ret) {
-        ++it;
+      // Trim segments
+      if (route_.begin_segment > 0) {
+        segments_.erase(segments_.begin(), segments_.lower_bound(route_.begin_segment));
+      }
+      if (route_.end_segment >= 0) {
+        segments_.erase(segments_.upper_bound(route_.end_segment), segments_.end());
       }
     }
+    return !segments_.empty();
+  }
+  return loadFromAutoSource();
+}
+
+bool Route::loadFromAutoSource() {
+  auto origin_prefix = getenv("OPENPILOT_PREFIX");
+  if (origin_prefix) {
+    setenv("OPENPILOT_PREFIX", "", 1);
+  }
+  auto cmd = util::string_format("python ../lib/logreader.py \"%s\" --identifiers-only", route_string_.c_str());
+  auto log_files = split(util::check_output(cmd), '\n');
+  if (origin_prefix) {
+    setenv("OPENPILOT_PREFIX", origin_prefix, 1);
+  }
+
+  for (int i = 0; i < log_files.size(); ++i) {
+    addFileToSegment(i, log_files[i]);
   }
   return !segments_.empty();
 }
@@ -155,6 +178,12 @@ void Route::addFileToSegment(int n, const std::string &file) {
   }
 }
 
+std::time_t Route::strToTime(const std::string &timestamp) {
+  struct tm tm_time = {0};
+  strptime(timestamp.c_str(), "%Y-%m-%d--%H-%M-%S", &tm_time);
+  return mktime(&tm_time);
+}
+
 // class Segment
 
 Segment::Segment(int n, const SegmentFile &files, uint32_t flags, const std::vector<bool> &filters,

tools/replay/route.h

@@ -24,6 +24,7 @@ enum class RouteLoadError {
 struct RouteIdentifier {
   std::string dongle_id;
   std::string timestamp;
+  std::time_t date_time;
   int begin_segment = 0;
   int end_segment = -1;
   std::string str;
@@ -40,11 +41,11 @@ struct SegmentFile {
 
 class Route {
 public:
-  Route(const std::string &route, const std::string &data_dir = {});
+  Route(const std::string &route, const std::string &data_dir = {}, bool auto_source = false);
   bool load();
   RouteLoadError lastError() const { return err_; }
   inline const std::string &name() const { return route_.str; }
-  inline const std::time_t datetime() const { return date_time_; }
+  inline const std::time_t datetime() const { return route_.date_time; }
   inline const std::string &dir() const { return data_dir_; }
   inline const RouteIdentifier &identifier() const { return route_; }
   inline const std::map<int, SegmentFile> &segments() const { return segments_; }
@@ -52,15 +53,19 @@ public:
   static RouteIdentifier parseRoute(const std::string &str);
 
 protected:
+  bool loadSegments();
+  bool loadFromAutoSource();
   bool loadFromLocal();
   bool loadFromServer(int retries = 3);
   bool loadFromJson(const std::string &json);
   void addFileToSegment(int seg_num, const std::string &file);
+  static std::time_t strToTime(const std::string &timestamp);
   RouteIdentifier route_ = {};
   std::string data_dir_;
   std::map<int, SegmentFile> segments_;
-  std::time_t date_time_;
   RouteLoadError err_ = RouteLoadError::None;
+  bool auto_source_ = false;
+  std::string route_string_;
 };
 
 class Segment {

tools/replay/seg_mgr.h

@@ -20,8 +20,8 @@ public:
     bool isSegmentLoaded(int n) const { return segments.find(n) != segments.end(); }
   };
 
-  SegmentManager(const std::string &route_name, uint32_t flags, const std::string &data_dir = "")
+  SegmentManager(const std::string &route_name, uint32_t flags, const std::string &data_dir = "", bool auto_source = false)
-      : flags_(flags), route_(route_name, data_dir), event_data_(std::make_shared<EventData>()) {}
+      : flags_(flags), route_(route_name, data_dir, auto_source), event_data_(std::make_shared<EventData>()) {}
   ~SegmentManager();
 
   bool load();