tools: add --auto flag to replay and cabana for loading routes from auto source (#34863)

* add flag to auto load a route from the most suitable source

* split to functions

* early return

* add --auto to replay

* README

* cleanup

* remove prefix

* parse datetime

* cleanup

* improve help

* do not modify logreader.py

* fix seg_num

* cleanup

tools/auto_source.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+import sys
+from openpilot.tools.lib.logreader import LogReader
+
+
+def main():
+  if len(sys.argv) != 2:
+    print("Usage: python auto_source.py <log_path>")
+    sys.exit(1)
+
+  log_path = sys.argv[1]
+  lr = LogReader(log_path, sort_by_time=True)
+  print("\n".join(lr.logreader_identifiers))
+
+
+if __name__ == "__main__":
+  main()

tools/cabana/README.md

@@ -12,6 +12,8 @@ Options:
   -h, --help                     Displays help on commandline options.
   --help-all                     Displays help including Qt specific options.
   --demo                         use a demo route instead of providing your own
+  --auto                         Auto load the route from the best available source (no video):
+                                 internal, openpilotci, comma_api, car_segments, testing_closet
   --qcam                         load qcamera
   --ecam                         load wide road camera
   --msgq                         read can messages from msgq

tools/cabana/cabana.cc

@@ -23,6 +23,7 @@ int main(int argc, char *argv[]) {
   cmd_parser.addHelpOption();
   cmd_parser.addPositionalArgument("route", "the drive to replay. find your drives at connect.comma.ai");
   cmd_parser.addOption({"demo", "use a demo route instead of providing your own"});
+  cmd_parser.addOption({"auto", "Auto load the route from the best available source (no video): internal, openpilotci, comma_api, car_segments, testing_closet"});
   cmd_parser.addOption({"qcam", "load qcamera"});
   cmd_parser.addOption({"ecam", "load wide road camera"});
   cmd_parser.addOption({"dcam", "load driver camera"});
@@ -69,7 +70,8 @@ int main(int argc, char *argv[]) {
     }
     if (!route.isEmpty()) {
       auto replay_stream = std::make_unique<ReplayStream>(&app);
-      if (!replay_stream->loadRoute(route, cmd_parser.value("data_dir"), replay_flags)) {
+      bool auto_source = cmd_parser.isSet("auto");
+      if (!replay_stream->loadRoute(route, cmd_parser.value("data_dir"), replay_flags, auto_source)) {
         return 0;
       }
       stream = replay_stream.release();

tools/cabana/streams/replaystream.cc

@@ -7,6 +7,7 @@
 #include <QPushButton>
 
 #include "common/timing.h"
+#include "common/util.h"
 #include "tools/cabana/streams/routes.h"
 
 ReplayStream::ReplayStream(QObject *parent) : AbstractStream(parent) {
@@ -45,9 +46,9 @@ void ReplayStream::mergeSegments() {
   }
 }
 
-bool ReplayStream::loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags) {
+bool ReplayStream::loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags, bool auto_source) {
   replay.reset(new Replay(route.toStdString(), {"can", "roadEncodeIdx", "driverEncodeIdx", "wideRoadEncodeIdx", "carParams"},
-                          {}, nullptr, replay_flags, data_dir.toStdString()));
+                          {}, nullptr, replay_flags, data_dir.toStdString(), auto_source));
   replay->setSegmentCacheLimit(settings.max_cached_minutes);
   replay->installEventFilter([this](const Event *event) { return eventFilter(event); });
 

tools/cabana/streams/replaystream.h

@@ -18,7 +18,7 @@ class ReplayStream : public AbstractStream {
 public:
   ReplayStream(QObject *parent);
   void start() override { replay->start(); }
-  bool loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags = REPLAY_FLAG_NONE);
+  bool loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags = REPLAY_FLAG_NONE, bool auto_source = false);
   bool eventFilter(const Event *event);
   void seekTo(double ts) override { replay->seekTo(std::max(double(0), ts), false); }
   bool liveStreaming() const override { return false; }

tools/replay/README.md

@@ -64,6 +64,8 @@ Options:
   -s, --start <seconds>  start from <seconds>
   -x <speed>             playback <speed>. between 0.2 - 3
   --demo                 use a demo route instead of providing your own
+  --auto                 Auto load the route from the best available source (no video):
+                         internal, openpilotci, comma_api, car_segments, testing_closet
   --data_dir <data_dir>  local directory with routes
   --prefix <prefix>      set OPENPILOT_PREFIX
   --dcam                 load driver camera

tools/replay/main.cc

@@ -19,6 +19,8 @@ Options:
   -s, --start        Start from <seconds>
   -x, --playback     Playback <speed>
       --demo         Use a demo route instead of providing your own
+      --auto         Auto load the route from the best available source (no video):
+                     internal, openpilotci, comma_api, car_segments, testing_closet
   -d, --data_dir     Local directory with routes
   -p, --prefix       Set OPENPILOT_PREFIX
       --dcam         Load driver camera
@@ -39,6 +41,7 @@ struct ReplayConfig {
   std::string data_dir;
   std::string prefix;
   uint32_t flags = REPLAY_FLAG_NONE;
+  bool auto_source = false;
   int start_seconds = 0;
   int cache_segments = -1;
   float playback_speed = -1;
@@ -52,6 +55,7 @@ bool parseArgs(int argc, char *argv[], ReplayConfig &config) {
       {"start", required_argument, nullptr, 's'},
       {"playback", required_argument, nullptr, 'x'},
       {"demo", no_argument, nullptr, 0},
+      {"auto", no_argument, nullptr, 0},
       {"data_dir", required_argument, nullptr, 'd'},
       {"prefix", required_argument, nullptr, 'p'},
       {"dcam", no_argument, nullptr, 0},
@@ -94,11 +98,9 @@ bool parseArgs(int argc, char *argv[], ReplayConfig &config) {
       case 'p': config.prefix = optarg; break;
       case 0: {
         std::string name = cli_options[option_index].name;
-        if (name == "demo") {
-          config.route = DEMO_ROUTE;
-        } else {
-          config.flags |= flag_map.at(name);
-        }
+        if (name == "demo") config.route = DEMO_ROUTE;
+        else if (name == "auto") config.auto_source = true;
+        else config.flags |= flag_map.at(name);
         break;
       }
       case 'h': std::cout << helpText; return false;
@@ -136,7 +138,7 @@ int main(int argc, char *argv[]) {
     op_prefix = std::make_unique<OpenpilotPrefix>(config.prefix);
   }
 
-  Replay replay(config.route, config.allow, config.block, nullptr, config.flags, config.data_dir);
+  Replay replay(config.route, config.allow, config.block, nullptr, config.flags, config.data_dir, config.auto_source);
   if (config.cache_segments > 0) {
     replay.setSegmentCacheLimit(config.cache_segments);
   }

tools/replay/replay.cc

@@ -15,8 +15,8 @@ void notifyEvent(Callback &callback, Args &&...args) {
 }
 
 Replay::Replay(const std::string &route, std::vector<std::string> allow, std::vector<std::string> block,
-               SubMaster *sm, uint32_t flags, const std::string &data_dir)
-    : sm_(sm), flags_(flags), seg_mgr_(std::make_unique<SegmentManager>(route, flags, data_dir)) {
+               SubMaster *sm, uint32_t flags, const std::string &data_dir, bool auto_source)
+    : sm_(sm), flags_(flags), seg_mgr_(std::make_unique<SegmentManager>(route, flags, data_dir, auto_source)) {
   std::signal(SIGUSR1, interrupt_sleep_handler);
 
   if (!(flags_ & REPLAY_FLAG_ALL_SERVICES)) {

tools/replay/replay.h

@@ -29,7 +29,7 @@ enum REPLAY_FLAGS {
 class Replay {
 public:
   Replay(const std::string &route, std::vector<std::string> allow, std::vector<std::string> block, SubMaster *sm = nullptr,
-         uint32_t flags = REPLAY_FLAG_NONE, const std::string &data_dir = "");
+         uint32_t flags = REPLAY_FLAG_NONE, const std::string &data_dir = "", bool auto_source = false);
   ~Replay();
   bool load();
   RouteLoadError lastRouteError() const { return route().lastError(); }

tools/replay/route.cc

@@ -10,9 +10,8 @@
 #include "tools/replay/replay.h"
 #include "tools/replay/util.h"
 
-Route::Route(const std::string &route, const std::string &data_dir) : data_dir_(data_dir) {
-  route_ = parseRoute(route);
-}
+Route::Route(const std::string &route, const std::string &data_dir, bool auto_source)
+    : route_string_(route), data_dir_(data_dir), auto_source_(auto_source) {}
 
 RouteIdentifier Route::parseRoute(const std::string &str) {
   RouteIdentifier identifier = {};
@@ -44,7 +43,7 @@ RouteIdentifier Route::parseRoute(const std::string &str) {
 }
 
 bool Route::load() {
-  err_ = RouteLoadError::None;
+  route_ = parseRoute(route_string_);
   if (route_.str.empty() || (data_dir_.empty() && route_.dongle_id.empty())) {
     rInfo("invalid route format");
     return false;
@@ -56,17 +55,50 @@ bool Route::load() {
     date_time_ = mktime(&tm_time);
   }
 
-  bool ret = data_dir_.empty() ? loadFromServer() : loadFromLocal();
-  if (ret) {
-    if (route_.begin_segment == -1) route_.begin_segment = segments_.rbegin()->first;
-    if (route_.end_segment == -1) route_.end_segment = segments_.rbegin()->first;
-    for (auto it = segments_.begin(); it != segments_.end(); /**/) {
-      if (it->first < route_.begin_segment || it->first > route_.end_segment) {
-        it = segments_.erase(it);
-      } else {
-        ++it;
+  if (!loadSegments()) {
+    rInfo("Failed to load segments");
+    return false;
+  }
+
+  return true;
+}
+
+bool Route::loadSegments() {
+  if (!auto_source_) {
+    bool ret = data_dir_.empty() ? loadFromServer() : loadFromLocal();
+    if (ret) {
+      // Trim segments
+      if (route_.begin_segment > 0) {
+        segments_.erase(segments_.begin(), segments_.lower_bound(route_.begin_segment));
       }
+      if (route_.end_segment >= 0) {
+        segments_.erase(segments_.upper_bound(route_.end_segment), segments_.end());
+      }
+    }
+    return !segments_.empty();
+  }
+  return loadFromAutoSource();
+}
+
+bool Route::loadFromAutoSource() {
+  auto origin_prefix = getenv("OPENPILOT_PREFIX");
+  if (origin_prefix) {
+    setenv("OPENPILOT_PREFIX", "", 1);
+  }
+  auto cmd = util::string_format("../auto_source.py \"%s\"", route_string_.c_str());
+  auto log_files = split(util::check_output(cmd), '\n');
+  if (origin_prefix) {
+    setenv("OPENPILOT_PREFIX", origin_prefix, 1);
+  }
+
+  const static std::regex rx(R"(\/(\d+)\/)");
+  for (int i = 0; i < log_files.size(); ++i) {
+    int seg_num = i;
+    std::smatch match;
+    if (std::regex_search(log_files[i], match, rx)) {
+      seg_num = std::stoi(match[1]);
     }
+    addFileToSegment(seg_num, log_files[i]);
   }
   return !segments_.empty();
 }

tools/replay/route.h

@@ -40,7 +40,7 @@ struct SegmentFile {
 
 class Route {
 public:
-  Route(const std::string &route, const std::string &data_dir = {});
+  Route(const std::string &route, const std::string &data_dir = {}, bool auto_source = false);
   bool load();
   RouteLoadError lastError() const { return err_; }
   inline const std::string &name() const { return route_.str; }
@@ -52,6 +52,8 @@ public:
   static RouteIdentifier parseRoute(const std::string &str);
 
 protected:
+  bool loadSegments();
+  bool loadFromAutoSource();
   bool loadFromLocal();
   bool loadFromServer(int retries = 3);
   bool loadFromJson(const std::string &json);
@@ -61,6 +63,8 @@ protected:
   std::map<int, SegmentFile> segments_;
   std::time_t date_time_ = 0;
   RouteLoadError err_ = RouteLoadError::None;
+  bool auto_source_ = false;
+  std::string route_string_;
 };
 
 class Segment {

tools/replay/seg_mgr.h

@@ -20,8 +20,8 @@ public:
     bool isSegmentLoaded(int n) const { return segments.find(n) != segments.end(); }
   };
 
-  SegmentManager(const std::string &route_name, uint32_t flags, const std::string &data_dir = "")
-      : flags_(flags), route_(route_name, data_dir), event_data_(std::make_shared<EventData>()) {}
+  SegmentManager(const std::string &route_name, uint32_t flags, const std::string &data_dir = "", bool auto_source = false)
+      : flags_(flags), route_(route_name, data_dir, auto_source), event_data_(std::make_shared<EventData>()) {}
   ~SegmentManager();
 
   bool load();