From b920e2a998eeb13e77fa383a53961b51a6247698 Mon Sep 17 00:00:00 2001 From: Dean Lee Date: Tue, 6 May 2025 07:48:15 +0800 Subject: [PATCH] tools: add --auto flag to replay and cabana for loading routes from auto source (#34863) * add flag to auto load a route from the most suitable source * split to functions * early return * add --auto to replay * README * cleanup * remove prefix * parse datetime * cleanup * improve help * do not modify logreader.py * fix seg_num * cleanup --- tools/auto_source.py | 17 ++++++++ tools/cabana/README.md | 2 + tools/cabana/cabana.cc | 4 +- tools/cabana/streams/replaystream.cc | 5 ++- tools/cabana/streams/replaystream.h | 2 +- tools/replay/README.md | 2 + tools/replay/main.cc | 14 ++++--- tools/replay/replay.cc | 4 +- tools/replay/replay.h | 2 +- tools/replay/route.cc | 58 +++++++++++++++++++++------- tools/replay/route.h | 6 ++- tools/replay/seg_mgr.h | 4 +- 12 files changed, 91 insertions(+), 29 deletions(-) create mode 100755 tools/auto_source.py diff --git a/tools/auto_source.py b/tools/auto_source.py new file mode 100755 index 0000000000..401929a9ad --- /dev/null +++ b/tools/auto_source.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +import sys +from openpilot.tools.lib.logreader import LogReader + + +def main(): + if len(sys.argv) != 2: + print("Usage: python auto_source.py ") + sys.exit(1) + + log_path = sys.argv[1] + lr = LogReader(log_path, sort_by_time=True) + print("\n".join(lr.logreader_identifiers)) + + +if __name__ == "__main__": + main() diff --git a/tools/cabana/README.md b/tools/cabana/README.md index 0b7c5bf3ee..7933098e34 100644 --- a/tools/cabana/README.md +++ b/tools/cabana/README.md @@ -12,6 +12,8 @@ Options: -h, --help Displays help on commandline options. --help-all Displays help including Qt specific options. --demo use a demo route instead of providing your own + --auto Auto load the route from the best available source (no video): + internal, openpilotci, comma_api, car_segments, testing_closet --qcam load qcamera --ecam load wide road camera --msgq read can messages from msgq diff --git a/tools/cabana/cabana.cc b/tools/cabana/cabana.cc index d140a323e1..97f178b1f3 100644 --- a/tools/cabana/cabana.cc +++ b/tools/cabana/cabana.cc @@ -23,6 +23,7 @@ int main(int argc, char *argv[]) { cmd_parser.addHelpOption(); cmd_parser.addPositionalArgument("route", "the drive to replay. find your drives at connect.comma.ai"); cmd_parser.addOption({"demo", "use a demo route instead of providing your own"}); + cmd_parser.addOption({"auto", "Auto load the route from the best available source (no video): internal, openpilotci, comma_api, car_segments, testing_closet"}); cmd_parser.addOption({"qcam", "load qcamera"}); cmd_parser.addOption({"ecam", "load wide road camera"}); cmd_parser.addOption({"dcam", "load driver camera"}); @@ -69,7 +70,8 @@ int main(int argc, char *argv[]) { } if (!route.isEmpty()) { auto replay_stream = std::make_unique(&app); - if (!replay_stream->loadRoute(route, cmd_parser.value("data_dir"), replay_flags)) { + bool auto_source = cmd_parser.isSet("auto"); + if (!replay_stream->loadRoute(route, cmd_parser.value("data_dir"), replay_flags, auto_source)) { return 0; } stream = replay_stream.release(); diff --git a/tools/cabana/streams/replaystream.cc b/tools/cabana/streams/replaystream.cc index ebe478ded7..b8cf1be299 100644 --- a/tools/cabana/streams/replaystream.cc +++ b/tools/cabana/streams/replaystream.cc @@ -7,6 +7,7 @@ #include #include "common/timing.h" +#include "common/util.h" #include "tools/cabana/streams/routes.h" ReplayStream::ReplayStream(QObject *parent) : AbstractStream(parent) { @@ -45,9 +46,9 @@ void ReplayStream::mergeSegments() { } } -bool ReplayStream::loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags) { +bool ReplayStream::loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags, bool auto_source) { replay.reset(new Replay(route.toStdString(), {"can", "roadEncodeIdx", "driverEncodeIdx", "wideRoadEncodeIdx", "carParams"}, - {}, nullptr, replay_flags, data_dir.toStdString())); + {}, nullptr, replay_flags, data_dir.toStdString(), auto_source)); replay->setSegmentCacheLimit(settings.max_cached_minutes); replay->installEventFilter([this](const Event *event) { return eventFilter(event); }); diff --git a/tools/cabana/streams/replaystream.h b/tools/cabana/streams/replaystream.h index df1f2526a5..d429ed1f95 100644 --- a/tools/cabana/streams/replaystream.h +++ b/tools/cabana/streams/replaystream.h @@ -18,7 +18,7 @@ class ReplayStream : public AbstractStream { public: ReplayStream(QObject *parent); void start() override { replay->start(); } - bool loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags = REPLAY_FLAG_NONE); + bool loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags = REPLAY_FLAG_NONE, bool auto_source = false); bool eventFilter(const Event *event); void seekTo(double ts) override { replay->seekTo(std::max(double(0), ts), false); } bool liveStreaming() const override { return false; } diff --git a/tools/replay/README.md b/tools/replay/README.md index 719f7d2db9..8b4afb0acc 100644 --- a/tools/replay/README.md +++ b/tools/replay/README.md @@ -64,6 +64,8 @@ Options: -s, --start start from -x playback . between 0.2 - 3 --demo use a demo route instead of providing your own + --auto Auto load the route from the best available source (no video): + internal, openpilotci, comma_api, car_segments, testing_closet --data_dir local directory with routes --prefix set OPENPILOT_PREFIX --dcam load driver camera diff --git a/tools/replay/main.cc b/tools/replay/main.cc index b33b7fa263..38a1da292a 100644 --- a/tools/replay/main.cc +++ b/tools/replay/main.cc @@ -19,6 +19,8 @@ Options: -s, --start Start from -x, --playback Playback --demo Use a demo route instead of providing your own + --auto Auto load the route from the best available source (no video): + internal, openpilotci, comma_api, car_segments, testing_closet -d, --data_dir Local directory with routes -p, --prefix Set OPENPILOT_PREFIX --dcam Load driver camera @@ -39,6 +41,7 @@ struct ReplayConfig { std::string data_dir; std::string prefix; uint32_t flags = REPLAY_FLAG_NONE; + bool auto_source = false; int start_seconds = 0; int cache_segments = -1; float playback_speed = -1; @@ -52,6 +55,7 @@ bool parseArgs(int argc, char *argv[], ReplayConfig &config) { {"start", required_argument, nullptr, 's'}, {"playback", required_argument, nullptr, 'x'}, {"demo", no_argument, nullptr, 0}, + {"auto", no_argument, nullptr, 0}, {"data_dir", required_argument, nullptr, 'd'}, {"prefix", required_argument, nullptr, 'p'}, {"dcam", no_argument, nullptr, 0}, @@ -94,11 +98,9 @@ bool parseArgs(int argc, char *argv[], ReplayConfig &config) { case 'p': config.prefix = optarg; break; case 0: { std::string name = cli_options[option_index].name; - if (name == "demo") { - config.route = DEMO_ROUTE; - } else { - config.flags |= flag_map.at(name); - } + if (name == "demo") config.route = DEMO_ROUTE; + else if (name == "auto") config.auto_source = true; + else config.flags |= flag_map.at(name); break; } case 'h': std::cout << helpText; return false; @@ -136,7 +138,7 @@ int main(int argc, char *argv[]) { op_prefix = std::make_unique(config.prefix); } - Replay replay(config.route, config.allow, config.block, nullptr, config.flags, config.data_dir); + Replay replay(config.route, config.allow, config.block, nullptr, config.flags, config.data_dir, config.auto_source); if (config.cache_segments > 0) { replay.setSegmentCacheLimit(config.cache_segments); } diff --git a/tools/replay/replay.cc b/tools/replay/replay.cc index 80f586daa6..a8e5cd9d43 100644 --- a/tools/replay/replay.cc +++ b/tools/replay/replay.cc @@ -15,8 +15,8 @@ void notifyEvent(Callback &callback, Args &&...args) { } Replay::Replay(const std::string &route, std::vector allow, std::vector block, - SubMaster *sm, uint32_t flags, const std::string &data_dir) - : sm_(sm), flags_(flags), seg_mgr_(std::make_unique(route, flags, data_dir)) { + SubMaster *sm, uint32_t flags, const std::string &data_dir, bool auto_source) + : sm_(sm), flags_(flags), seg_mgr_(std::make_unique(route, flags, data_dir, auto_source)) { std::signal(SIGUSR1, interrupt_sleep_handler); if (!(flags_ & REPLAY_FLAG_ALL_SERVICES)) { diff --git a/tools/replay/replay.h b/tools/replay/replay.h index 6a2c86ff02..5e868d2427 100644 --- a/tools/replay/replay.h +++ b/tools/replay/replay.h @@ -29,7 +29,7 @@ enum REPLAY_FLAGS { class Replay { public: Replay(const std::string &route, std::vector allow, std::vector block, SubMaster *sm = nullptr, - uint32_t flags = REPLAY_FLAG_NONE, const std::string &data_dir = ""); + uint32_t flags = REPLAY_FLAG_NONE, const std::string &data_dir = "", bool auto_source = false); ~Replay(); bool load(); RouteLoadError lastRouteError() const { return route().lastError(); } diff --git a/tools/replay/route.cc b/tools/replay/route.cc index 98fa0e290e..ba00828267 100644 --- a/tools/replay/route.cc +++ b/tools/replay/route.cc @@ -10,9 +10,8 @@ #include "tools/replay/replay.h" #include "tools/replay/util.h" -Route::Route(const std::string &route, const std::string &data_dir) : data_dir_(data_dir) { - route_ = parseRoute(route); -} +Route::Route(const std::string &route, const std::string &data_dir, bool auto_source) + : route_string_(route), data_dir_(data_dir), auto_source_(auto_source) {} RouteIdentifier Route::parseRoute(const std::string &str) { RouteIdentifier identifier = {}; @@ -44,7 +43,7 @@ RouteIdentifier Route::parseRoute(const std::string &str) { } bool Route::load() { - err_ = RouteLoadError::None; + route_ = parseRoute(route_string_); if (route_.str.empty() || (data_dir_.empty() && route_.dongle_id.empty())) { rInfo("invalid route format"); return false; @@ -56,17 +55,50 @@ bool Route::load() { date_time_ = mktime(&tm_time); } - bool ret = data_dir_.empty() ? loadFromServer() : loadFromLocal(); - if (ret) { - if (route_.begin_segment == -1) route_.begin_segment = segments_.rbegin()->first; - if (route_.end_segment == -1) route_.end_segment = segments_.rbegin()->first; - for (auto it = segments_.begin(); it != segments_.end(); /**/) { - if (it->first < route_.begin_segment || it->first > route_.end_segment) { - it = segments_.erase(it); - } else { - ++it; + if (!loadSegments()) { + rInfo("Failed to load segments"); + return false; + } + + return true; +} + +bool Route::loadSegments() { + if (!auto_source_) { + bool ret = data_dir_.empty() ? loadFromServer() : loadFromLocal(); + if (ret) { + // Trim segments + if (route_.begin_segment > 0) { + segments_.erase(segments_.begin(), segments_.lower_bound(route_.begin_segment)); } + if (route_.end_segment >= 0) { + segments_.erase(segments_.upper_bound(route_.end_segment), segments_.end()); + } + } + return !segments_.empty(); + } + return loadFromAutoSource(); +} + +bool Route::loadFromAutoSource() { + auto origin_prefix = getenv("OPENPILOT_PREFIX"); + if (origin_prefix) { + setenv("OPENPILOT_PREFIX", "", 1); + } + auto cmd = util::string_format("../auto_source.py \"%s\"", route_string_.c_str()); + auto log_files = split(util::check_output(cmd), '\n'); + if (origin_prefix) { + setenv("OPENPILOT_PREFIX", origin_prefix, 1); + } + + const static std::regex rx(R"(\/(\d+)\/)"); + for (int i = 0; i < log_files.size(); ++i) { + int seg_num = i; + std::smatch match; + if (std::regex_search(log_files[i], match, rx)) { + seg_num = std::stoi(match[1]); } + addFileToSegment(seg_num, log_files[i]); } return !segments_.empty(); } diff --git a/tools/replay/route.h b/tools/replay/route.h index fb9f1869f6..0375252a19 100644 --- a/tools/replay/route.h +++ b/tools/replay/route.h @@ -40,7 +40,7 @@ struct SegmentFile { class Route { public: - Route(const std::string &route, const std::string &data_dir = {}); + Route(const std::string &route, const std::string &data_dir = {}, bool auto_source = false); bool load(); RouteLoadError lastError() const { return err_; } inline const std::string &name() const { return route_.str; } @@ -52,6 +52,8 @@ public: static RouteIdentifier parseRoute(const std::string &str); protected: + bool loadSegments(); + bool loadFromAutoSource(); bool loadFromLocal(); bool loadFromServer(int retries = 3); bool loadFromJson(const std::string &json); @@ -61,6 +63,8 @@ protected: std::map segments_; std::time_t date_time_ = 0; RouteLoadError err_ = RouteLoadError::None; + bool auto_source_ = false; + std::string route_string_; }; class Segment { diff --git a/tools/replay/seg_mgr.h b/tools/replay/seg_mgr.h index 9158e41618..640169749e 100644 --- a/tools/replay/seg_mgr.h +++ b/tools/replay/seg_mgr.h @@ -20,8 +20,8 @@ public: bool isSegmentLoaded(int n) const { return segments.find(n) != segments.end(); } }; - SegmentManager(const std::string &route_name, uint32_t flags, const std::string &data_dir = "") - : flags_(flags), route_(route_name, data_dir), event_data_(std::make_shared()) {} + SegmentManager(const std::string &route_name, uint32_t flags, const std::string &data_dir = "", bool auto_source = false) + : flags_(flags), route_(route_name, data_dir, auto_source), event_data_(std::make_shared()) {} ~SegmentManager(); bool load();