openpilot_comma/tools/replay/route.cc

#include "tools/replay/route.h"

#include <array>
#include <filesystem>
#include <regex>

#include "third_party/json11/json11.hpp"
#include "system/hardware/hw.h"
#include "tools/replay/api.h"
#include "tools/replay/replay.h"
#include "tools/replay/util.h"

Route::Route(const std::string &route, const std::string &data_dir, bool auto_source)
    : route_string_(route), data_dir_(data_dir), auto_source_(auto_source) {}

RouteIdentifier Route::parseRoute(const std::string &str) {
  RouteIdentifier identifier = {};
  static const std::regex pattern(R"(^(([a-z0-9]{16})[|_/])?(.{20})((--|/)((-?\d+(:(-?\d+)?)?)|(:-?\d+)))?$)");
  std::smatch match;

  if (std::regex_match(str, match, pattern)) {
    identifier.dongle_id = match[2].str();
    identifier.timestamp = match[3].str();
    identifier.str = identifier.dongle_id + "|" + identifier.timestamp;

    const auto separator = match[5].str();
    const auto range_str = match[6].str();
    if (!range_str.empty()) {
      if (separator == "/") {
        int pos = range_str.find(':');
        int begin_seg = std::stoi(range_str.substr(0, pos));
        identifier.begin_segment = identifier.end_segment = begin_seg;
        if (pos != std::string::npos) {
          auto end_seg_str = range_str.substr(pos + 1);
          identifier.end_segment = end_seg_str.empty() ? -1 : std::stoi(end_seg_str);
        }
      } else if (separator == "--") {
        identifier.begin_segment = std::atoi(range_str.c_str());
      }
    }
  }
  return identifier;
}

bool Route::load() {
  route_ = parseRoute(route_string_);
  if (route_.str.empty() || (data_dir_.empty() && route_.dongle_id.empty())) {
    rInfo("invalid route format");
    return false;
  }

  // Parse the timestamp from the route identifier (only applicable for old route formats).
  struct tm tm_time = {0};
  if (strptime(route_.timestamp.c_str(), "%Y-%m-%d--%H-%M-%S", &tm_time)) {
    date_time_ = mktime(&tm_time);
  }

  if (!loadSegments()) {
    rInfo("Failed to load segments");
    return false;
  }

  return true;
}

bool Route::loadSegments() {
  if (!auto_source_) {
    bool ret = data_dir_.empty() ? loadFromServer() : loadFromLocal();
    if (ret) {
      // Trim segments
      if (route_.begin_segment > 0) {
        segments_.erase(segments_.begin(), segments_.lower_bound(route_.begin_segment));
      }
      if (route_.end_segment >= 0) {
        segments_.erase(segments_.upper_bound(route_.end_segment), segments_.end());
      }
    }
    return !segments_.empty();
  }
  return loadFromAutoSource();
}

bool Route::loadFromAutoSource() {
  auto origin_prefix = getenv("OPENPILOT_PREFIX");
  if (origin_prefix) {
    setenv("OPENPILOT_PREFIX", "", 1);
  }
  auto cmd = util::string_format("../auto_source.py \"%s\"", route_string_.c_str());
  auto log_files = split(util::check_output(cmd), '\n');
  if (origin_prefix) {
    setenv("OPENPILOT_PREFIX", origin_prefix, 1);
  }

  const static std::regex rx(R"(\/(\d+)\/)");
  for (int i = 0; i < log_files.size(); ++i) {
    int seg_num = i;
    std::smatch match;
    if (std::regex_search(log_files[i], match, rx)) {
      seg_num = std::stoi(match[1]);
    }
    addFileToSegment(seg_num, log_files[i]);
  }
  return !segments_.empty();
}

bool Route::loadFromServer(int retries) {
  const std::string url = CommaApi2::BASE_URL + "/v1/route/" + route_.str + "/files";
  for (int i = 1; i <= retries; ++i) {
    long response_code = 0;
    std::string result = CommaApi2::httpGet(url, &response_code);
    if (response_code == 200) {
      return loadFromJson(result);
    }

    if (response_code == 401 || response_code == 403) {
      rWarning(">> Unauthorized. Authenticate with tools/lib/auth.py <<");
      err_ = RouteLoadError::Unauthorized;
      break;
    }
    if (response_code == 404) {
      rWarning("The specified route could not be found on the server.");
      err_ = RouteLoadError::FileNotFound;
      break;
    }

    err_ = RouteLoadError::NetworkError;
    rWarning("Retrying %d/%d", i, retries);
    util::sleep_for(3000);
  }

  return false;
}

bool Route::loadFromJson(const std::string &json) {
  const static std::regex rx(R"(\/(\d+)\/)");
  std::string err;
  auto jsonData = json11::Json::parse(json, err);
  if (!err.empty()) {
    rWarning("JSON parsing error: %s", err.c_str());
    return false;
  }
  for (const auto &value : jsonData.object_items()) {
    const auto &urlArray = value.second.array_items();
    for (const auto &url : urlArray) {
      std::string url_str = url.string_value();
      std::smatch match;
      if (std::regex_search(url_str, match, rx)) {
        addFileToSegment(std::stoi(match[1]), url_str);
      }
    }
  }
  return !segments_.empty();
}

bool Route::loadFromLocal() {
  std::string pattern = route_.timestamp + "--";
  for (const auto &entry : std::filesystem::directory_iterator(data_dir_)) {
    if (entry.is_directory() && entry.path().filename().string().find(pattern) != std::string::npos) {
      std::string segment = entry.path().string();
      int seg_num = std::atoi(segment.substr(segment.rfind("--") + 2).c_str());

      for (const auto &file : std::filesystem::directory_iterator(segment)) {
        if (file.is_regular_file()) {
          addFileToSegment(seg_num, file.path().string());
        }
      }
    }
  }
  return !segments_.empty();
}

void Route::addFileToSegment(int n, const std::string &file) {
  std::string name = extractFileName(file);
  auto pos = name.find_last_of("--");
  name = pos != std::string::npos ? name.substr(pos + 2) : name;

  if (name == "rlog.bz2" || name == "rlog.zst" || name == "rlog") {
    segments_[n].rlog = file;
  } else if (name == "qlog.bz2" || name == "qlog.zst" || name == "qlog") {
    segments_[n].qlog = file;
  } else if (name == "fcamera.hevc") {
    segments_[n].road_cam = file;
  } else if (name == "dcamera.hevc") {
    segments_[n].driver_cam = file;
  } else if (name == "ecamera.hevc") {
    segments_[n].wide_road_cam = file;
  } else if (name == "qcamera.ts") {
    segments_[n].qcamera = file;
  }
}

// class Segment

Segment::Segment(int n, const SegmentFile &files, uint32_t flags, const std::vector<bool> &filters,
                 std::function<void(int, bool)> callback)
    : seg_num(n), flags(flags), filters_(filters), on_load_finished_(callback) {
  // [RoadCam, DriverCam, WideRoadCam, log]. fallback to qcamera/qlog
  const std::array file_list = {
      (flags & REPLAY_FLAG_QCAMERA) || files.road_cam.empty() ? files.qcamera : files.road_cam,
      flags & REPLAY_FLAG_DCAM ? files.driver_cam : "",
      flags & REPLAY_FLAG_ECAM ? files.wide_road_cam : "",
      files.rlog.empty() ? files.qlog : files.rlog,
  };
  for (int i = 0; i < file_list.size(); ++i) {
    if (!file_list[i].empty() && (!(flags & REPLAY_FLAG_NO_VIPC) || i >= MAX_CAMERAS)) {
      ++loading_;
      threads_.emplace_back(&Segment::loadFile, this, i, file_list[i]);
    }
  }
}

Segment::~Segment() {
  {
    std::lock_guard lock(mutex_);
    on_load_finished_ = nullptr;  // Prevent callback after destruction
  }
  abort_ = true;
  for (auto &thread : threads_) {
    if (thread.joinable()) thread.join();
  }
}

void Segment::loadFile(int id, const std::string file) {
  const bool local_cache = !(flags & REPLAY_FLAG_NO_FILE_CACHE);
  bool success = false;
  if (id < MAX_CAMERAS) {
    frames[id] = std::make_unique<FrameReader>();
    success = frames[id]->load((CameraType)id, file, flags & REPLAY_FLAG_NO_HW_DECODER, &abort_, local_cache, 20 * 1024 * 1024, 3);
  } else {
    log = std::make_unique<LogReader>(filters_);
    success = log->load(file, &abort_, local_cache, 0, 3);
  }

  if (!success) {
    // abort all loading jobs.
    abort_ = true;
  }

  if (--loading_ == 0) {
    std::lock_guard lock(mutex_);
    load_state_ = !abort_ ? LoadState::Loaded : LoadState::Failed;
    if (on_load_finished_) {
      on_load_finished_(seg_num, !abort_);
    }
  }
}

Segment::LoadState Segment::getState() {
  std::scoped_lock lock(mutex_);
  return load_state_;
}