diff --git a/tools/cabana/README.md b/tools/cabana/README.md index 0b7c5bf3ee..7933098e34 100644 --- a/tools/cabana/README.md +++ b/tools/cabana/README.md @@ -12,6 +12,8 @@ Options: -h, --help Displays help on commandline options. --help-all Displays help including Qt specific options. --demo use a demo route instead of providing your own + --auto Auto load the route from the best available source (no video): + internal, openpilotci, comma_api, car_segments, testing_closet --qcam load qcamera --ecam load wide road camera --msgq read can messages from msgq diff --git a/tools/cabana/cabana.cc b/tools/cabana/cabana.cc index d140a323e1..97f178b1f3 100644 --- a/tools/cabana/cabana.cc +++ b/tools/cabana/cabana.cc @@ -23,6 +23,7 @@ int main(int argc, char *argv[]) { cmd_parser.addHelpOption(); cmd_parser.addPositionalArgument("route", "the drive to replay. find your drives at connect.comma.ai"); cmd_parser.addOption({"demo", "use a demo route instead of providing your own"}); + cmd_parser.addOption({"auto", "Auto load the route from the best available source (no video): internal, openpilotci, comma_api, car_segments, testing_closet"}); cmd_parser.addOption({"qcam", "load qcamera"}); cmd_parser.addOption({"ecam", "load wide road camera"}); cmd_parser.addOption({"dcam", "load driver camera"}); @@ -69,7 +70,8 @@ int main(int argc, char *argv[]) { } if (!route.isEmpty()) { auto replay_stream = std::make_unique(&app); - if (!replay_stream->loadRoute(route, cmd_parser.value("data_dir"), replay_flags)) { + bool auto_source = cmd_parser.isSet("auto"); + if (!replay_stream->loadRoute(route, cmd_parser.value("data_dir"), replay_flags, auto_source)) { return 0; } stream = replay_stream.release(); diff --git a/tools/cabana/streams/replaystream.cc b/tools/cabana/streams/replaystream.cc index ebe478ded7..b8cf1be299 100644 --- a/tools/cabana/streams/replaystream.cc +++ b/tools/cabana/streams/replaystream.cc @@ -7,6 +7,7 @@ #include #include "common/timing.h" +#include "common/util.h" #include "tools/cabana/streams/routes.h" ReplayStream::ReplayStream(QObject *parent) : AbstractStream(parent) { @@ -45,9 +46,9 @@ void ReplayStream::mergeSegments() { } } -bool ReplayStream::loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags) { +bool ReplayStream::loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags, bool auto_source) { replay.reset(new Replay(route.toStdString(), {"can", "roadEncodeIdx", "driverEncodeIdx", "wideRoadEncodeIdx", "carParams"}, - {}, nullptr, replay_flags, data_dir.toStdString())); + {}, nullptr, replay_flags, data_dir.toStdString(), auto_source)); replay->setSegmentCacheLimit(settings.max_cached_minutes); replay->installEventFilter([this](const Event *event) { return eventFilter(event); }); diff --git a/tools/cabana/streams/replaystream.h b/tools/cabana/streams/replaystream.h index df1f2526a5..d429ed1f95 100644 --- a/tools/cabana/streams/replaystream.h +++ b/tools/cabana/streams/replaystream.h @@ -18,7 +18,7 @@ class ReplayStream : public AbstractStream { public: ReplayStream(QObject *parent); void start() override { replay->start(); } - bool loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags = REPLAY_FLAG_NONE); + bool loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags = REPLAY_FLAG_NONE, bool auto_source = false); bool eventFilter(const Event *event); void seekTo(double ts) override { replay->seekTo(std::max(double(0), ts), false); } bool liveStreaming() const override { return false; } diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index 34d3e5ea9f..763ef35cd6 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -6,12 +6,12 @@ import capnp import enum import os import pathlib -import sys import tqdm import urllib.parse import warnings import zstandard as zstd +from argparse import ArgumentParser from collections.abc import Callable, Iterable, Iterator from urllib.parse import parse_qs, urlparse @@ -333,7 +333,15 @@ if __name__ == "__main__": # capnproto <= 0.8.0 throws errors converting byte data to string # below line catches those errors and replaces the bytes with \x__ codecs.register_error("strict", codecs.backslashreplace_errors) - log_path = sys.argv[1] - lr = LogReader(log_path, sort_by_time=True) - for msg in lr: - print(msg) + + parser = ArgumentParser(description="Process a log file and print identifiers or full messages.") + parser.add_argument("log_path", help="Path to the log file") + parser.add_argument("--identifiers-only", action="store_true", help="Print only log identifiers") + args = parser.parse_args() + + lr = LogReader(args.log_path, sort_by_time=True) + if args.identifiers_only: + print("\n".join(lr.logreader_identifiers)) + else: + for msg in lr: + print(msg) diff --git a/tools/replay/README.md b/tools/replay/README.md index f02090c58e..4a09969bd3 100644 --- a/tools/replay/README.md +++ b/tools/replay/README.md @@ -64,6 +64,8 @@ Options: -s, --start start from -x playback . between 0.2 - 3 --demo use a demo route instead of providing your own + --auto Auto load the route from the best available source (no video): + internal, openpilotci, comma_api, car_segments, testing_closet --data_dir local directory with routes --prefix set OPENPILOT_PREFIX --dcam load driver camera diff --git a/tools/replay/main.cc b/tools/replay/main.cc index b33b7fa263..38a1da292a 100644 --- a/tools/replay/main.cc +++ b/tools/replay/main.cc @@ -19,6 +19,8 @@ Options: -s, --start Start from -x, --playback Playback --demo Use a demo route instead of providing your own + --auto Auto load the route from the best available source (no video): + internal, openpilotci, comma_api, car_segments, testing_closet -d, --data_dir Local directory with routes -p, --prefix Set OPENPILOT_PREFIX --dcam Load driver camera @@ -39,6 +41,7 @@ struct ReplayConfig { std::string data_dir; std::string prefix; uint32_t flags = REPLAY_FLAG_NONE; + bool auto_source = false; int start_seconds = 0; int cache_segments = -1; float playback_speed = -1; @@ -52,6 +55,7 @@ bool parseArgs(int argc, char *argv[], ReplayConfig &config) { {"start", required_argument, nullptr, 's'}, {"playback", required_argument, nullptr, 'x'}, {"demo", no_argument, nullptr, 0}, + {"auto", no_argument, nullptr, 0}, {"data_dir", required_argument, nullptr, 'd'}, {"prefix", required_argument, nullptr, 'p'}, {"dcam", no_argument, nullptr, 0}, @@ -94,11 +98,9 @@ bool parseArgs(int argc, char *argv[], ReplayConfig &config) { case 'p': config.prefix = optarg; break; case 0: { std::string name = cli_options[option_index].name; - if (name == "demo") { - config.route = DEMO_ROUTE; - } else { - config.flags |= flag_map.at(name); - } + if (name == "demo") config.route = DEMO_ROUTE; + else if (name == "auto") config.auto_source = true; + else config.flags |= flag_map.at(name); break; } case 'h': std::cout << helpText; return false; @@ -136,7 +138,7 @@ int main(int argc, char *argv[]) { op_prefix = std::make_unique(config.prefix); } - Replay replay(config.route, config.allow, config.block, nullptr, config.flags, config.data_dir); + Replay replay(config.route, config.allow, config.block, nullptr, config.flags, config.data_dir, config.auto_source); if (config.cache_segments > 0) { replay.setSegmentCacheLimit(config.cache_segments); } diff --git a/tools/replay/replay.cc b/tools/replay/replay.cc index 80f586daa6..a8e5cd9d43 100644 --- a/tools/replay/replay.cc +++ b/tools/replay/replay.cc @@ -15,8 +15,8 @@ void notifyEvent(Callback &callback, Args &&...args) { } Replay::Replay(const std::string &route, std::vector allow, std::vector block, - SubMaster *sm, uint32_t flags, const std::string &data_dir) - : sm_(sm), flags_(flags), seg_mgr_(std::make_unique(route, flags, data_dir)) { + SubMaster *sm, uint32_t flags, const std::string &data_dir, bool auto_source) + : sm_(sm), flags_(flags), seg_mgr_(std::make_unique(route, flags, data_dir, auto_source)) { std::signal(SIGUSR1, interrupt_sleep_handler); if (!(flags_ & REPLAY_FLAG_ALL_SERVICES)) { diff --git a/tools/replay/replay.h b/tools/replay/replay.h index 6a2c86ff02..5e868d2427 100644 --- a/tools/replay/replay.h +++ b/tools/replay/replay.h @@ -29,7 +29,7 @@ enum REPLAY_FLAGS { class Replay { public: Replay(const std::string &route, std::vector allow, std::vector block, SubMaster *sm = nullptr, - uint32_t flags = REPLAY_FLAG_NONE, const std::string &data_dir = ""); + uint32_t flags = REPLAY_FLAG_NONE, const std::string &data_dir = "", bool auto_source = false); ~Replay(); bool load(); RouteLoadError lastRouteError() const { return route().lastError(); } diff --git a/tools/replay/route.cc b/tools/replay/route.cc index 7731d0daf4..47d9b66ad6 100644 --- a/tools/replay/route.cc +++ b/tools/replay/route.cc @@ -10,9 +10,8 @@ #include "tools/replay/replay.h" #include "tools/replay/util.h" -Route::Route(const std::string &route, const std::string &data_dir) : data_dir_(data_dir) { - route_ = parseRoute(route); -} +Route::Route(const std::string &route, const std::string &data_dir, bool auto_source) + : route_string_(route), data_dir_(data_dir), auto_source_(auto_source) {} RouteIdentifier Route::parseRoute(const std::string &str) { RouteIdentifier identifier = {}; @@ -22,6 +21,7 @@ RouteIdentifier Route::parseRoute(const std::string &str) { if (std::regex_match(str, match, pattern)) { identifier.dongle_id = match[2].str(); identifier.timestamp = match[3].str(); + identifier.date_time = strToTime(identifier.timestamp); identifier.str = identifier.dongle_id + "|" + identifier.timestamp; const auto separator = match[5].str(); @@ -44,27 +44,50 @@ RouteIdentifier Route::parseRoute(const std::string &str) { } bool Route::load() { - err_ = RouteLoadError::None; + route_ = parseRoute(route_string_); if (route_.str.empty() || (data_dir_.empty() && route_.dongle_id.empty())) { rInfo("invalid route format"); return false; } - struct tm tm_time = {0}; - strptime(route_.timestamp.c_str(), "%Y-%m-%d--%H-%M-%S", &tm_time); - date_time_ = mktime(&tm_time); - - bool ret = data_dir_.empty() ? loadFromServer() : loadFromLocal(); - if (ret) { - if (route_.begin_segment == -1) route_.begin_segment = segments_.rbegin()->first; - if (route_.end_segment == -1) route_.end_segment = segments_.rbegin()->first; - for (auto it = segments_.begin(); it != segments_.end(); /**/) { - if (it->first < route_.begin_segment || it->first > route_.end_segment) { - it = segments_.erase(it); - } else { - ++it; + if (!loadSegments()) { + rInfo("Failed to load segments"); + return false; + } + + return true; +} + +bool Route::loadSegments() { + if (!auto_source_) { + bool ret = data_dir_.empty() ? loadFromServer() : loadFromLocal(); + if (ret) { + // Trim segments + if (route_.begin_segment > 0) { + segments_.erase(segments_.begin(), segments_.lower_bound(route_.begin_segment)); + } + if (route_.end_segment >= 0) { + segments_.erase(segments_.upper_bound(route_.end_segment), segments_.end()); } } + return !segments_.empty(); + } + return loadFromAutoSource(); +} + +bool Route::loadFromAutoSource() { + auto origin_prefix = getenv("OPENPILOT_PREFIX"); + if (origin_prefix) { + setenv("OPENPILOT_PREFIX", "", 1); + } + auto cmd = util::string_format("python ../lib/logreader.py \"%s\" --identifiers-only", route_string_.c_str()); + auto log_files = split(util::check_output(cmd), '\n'); + if (origin_prefix) { + setenv("OPENPILOT_PREFIX", origin_prefix, 1); + } + + for (int i = 0; i < log_files.size(); ++i) { + addFileToSegment(i, log_files[i]); } return !segments_.empty(); } @@ -155,6 +178,12 @@ void Route::addFileToSegment(int n, const std::string &file) { } } +std::time_t Route::strToTime(const std::string ×tamp) { + struct tm tm_time = {0}; + strptime(timestamp.c_str(), "%Y-%m-%d--%H-%M-%S", &tm_time); + return mktime(&tm_time); +} + // class Segment Segment::Segment(int n, const SegmentFile &files, uint32_t flags, const std::vector &filters, diff --git a/tools/replay/route.h b/tools/replay/route.h index 1806be5afa..6226a9d64c 100644 --- a/tools/replay/route.h +++ b/tools/replay/route.h @@ -24,6 +24,7 @@ enum class RouteLoadError { struct RouteIdentifier { std::string dongle_id; std::string timestamp; + std::time_t date_time; int begin_segment = 0; int end_segment = -1; std::string str; @@ -40,11 +41,11 @@ struct SegmentFile { class Route { public: - Route(const std::string &route, const std::string &data_dir = {}); + Route(const std::string &route, const std::string &data_dir = {}, bool auto_source = false); bool load(); RouteLoadError lastError() const { return err_; } inline const std::string &name() const { return route_.str; } - inline const std::time_t datetime() const { return date_time_; } + inline const std::time_t datetime() const { return route_.date_time; } inline const std::string &dir() const { return data_dir_; } inline const RouteIdentifier &identifier() const { return route_; } inline const std::map &segments() const { return segments_; } @@ -52,15 +53,19 @@ public: static RouteIdentifier parseRoute(const std::string &str); protected: + bool loadSegments(); + bool loadFromAutoSource(); bool loadFromLocal(); bool loadFromServer(int retries = 3); bool loadFromJson(const std::string &json); void addFileToSegment(int seg_num, const std::string &file); + static std::time_t strToTime(const std::string ×tamp); RouteIdentifier route_ = {}; std::string data_dir_; std::map segments_; - std::time_t date_time_; RouteLoadError err_ = RouteLoadError::None; + bool auto_source_ = false; + std::string route_string_; }; class Segment { diff --git a/tools/replay/seg_mgr.h b/tools/replay/seg_mgr.h index 9158e41618..640169749e 100644 --- a/tools/replay/seg_mgr.h +++ b/tools/replay/seg_mgr.h @@ -20,8 +20,8 @@ public: bool isSegmentLoaded(int n) const { return segments.find(n) != segments.end(); } }; - SegmentManager(const std::string &route_name, uint32_t flags, const std::string &data_dir = "") - : flags_(flags), route_(route_name, data_dir), event_data_(std::make_shared()) {} + SegmentManager(const std::string &route_name, uint32_t flags, const std::string &data_dir = "", bool auto_source = false) + : flags_(flags), route_(route_name, data_dir, auto_source), event_data_(std::make_shared()) {} ~SegmentManager(); bool load();