replay: improve segment download and merge (#22654)
* no-cache mode * fix test cases build error * space * don't create cache dir in no-cache mode * fix errors in test cases * no_local_cache_ * set the number of connections by chunk_size * use size_t instead of int64_t * add test case for no-cache mode * rename variables * fix SIGSEGV * cleanup * faster decompressBZ2 * always decompress bz2 * add test cases * prepare for python interface * fix test cases build error * continue * camera_replay: cache remote file * protected inheritance * single option name * TODO * test_case for LogReader&FrameReader * fix wrong require * test case for FileReader * cleanup test * test:fix wrong filename * check cached file's checksum * fix mkdir permissions err cleanup filereader * remove initialize libav network libraries. dd * abort all loading if one failed * cleanup tests * use threadpool to limit concurrent downloads * cache more segments * merge 3 segments for replay * one segment uses about 100M of memory * use segments_need_merge.size() * shutdown * fix stuck if exit replay before keyboard thread started * load one segment at a time * small cleanup * cleanup filereader * space * tiny cleanup * merge master * cleanup test cases * use util:create_directories * cleanup framereaderpull/214/head
Dean Lee
4 years ago
committed by
GitHub
parent
5b93d44459
commit
2b4a477fbc
19 changed files with 347 additions and 237 deletions
@ -0,0 +1,62 @@ |
|||||||
|
#include "selfdrive/ui/replay/filereader.h" |
||||||
|
|
||||||
|
#include <sys/stat.h> |
||||||
|
|
||||||
|
#include <cassert> |
||||||
|
#include <cmath> |
||||||
|
#include <fstream> |
||||||
|
#include <iostream> |
||||||
|
#include <sstream> |
||||||
|
|
||||||
|
#include "selfdrive/common/util.h" |
||||||
|
#include "selfdrive/ui/replay/util.h" |
||||||
|
|
||||||
|
std::string cacheFilePath(const std::string &url) { |
||||||
|
static std::string cache_path = [] { |
||||||
|
const std::string comma_cache = util::getenv("COMMA_CACHE", "/tmp/comma_download_cache/"); |
||||||
|
util::create_directories(comma_cache, 0755); |
||||||
|
return comma_cache.back() == '/' ? comma_cache : comma_cache + "/"; |
||||||
|
}(); |
||||||
|
|
||||||
|
return cache_path + sha256(getUrlWithoutQuery(url));; |
||||||
|
} |
||||||
|
|
||||||
|
std::string FileReader::read(const std::string &file, std::atomic<bool> *abort) { |
||||||
|
const bool is_remote = file.find("https://") == 0; |
||||||
|
const std::string local_file = is_remote ? cacheFilePath(file) : file; |
||||||
|
std::string result; |
||||||
|
|
||||||
|
if ((!is_remote || cache_to_local_) && util::file_exists(local_file)) { |
||||||
|
result = util::read_file(local_file); |
||||||
|
} else if (is_remote) { |
||||||
|
result = download(file, abort); |
||||||
|
if (cache_to_local_ && !result.empty()) { |
||||||
|
std::ofstream fs(local_file, fs.binary | fs.out); |
||||||
|
fs.write(result.data(), result.size()); |
||||||
|
} |
||||||
|
} |
||||||
|
return result; |
||||||
|
} |
||||||
|
|
||||||
|
std::string FileReader::download(const std::string &url, std::atomic<bool> *abort) { |
||||||
|
std::string result; |
||||||
|
size_t remote_file_size = 0; |
||||||
|
for (int i = 0; i <= max_retries_ && !(abort && *abort); ++i) { |
||||||
|
if (i > 0) { |
||||||
|
std::cout << "download failed, retrying" << i << std::endl; |
||||||
|
} |
||||||
|
if (remote_file_size <= 0) { |
||||||
|
remote_file_size = getRemoteFileSize(url); |
||||||
|
} |
||||||
|
if (remote_file_size > 0 && !(abort && *abort)) { |
||||||
|
std::ostringstream oss; |
||||||
|
result.resize(remote_file_size); |
||||||
|
oss.rdbuf()->pubsetbuf(result.data(), result.size()); |
||||||
|
int chunks = chunk_size_ > 0 ? std::min(1, (int)std::nearbyint(remote_file_size / (float)chunk_size_)) : 1; |
||||||
|
if (httpMultiPartDownload(url, oss, chunks, remote_file_size, abort)) { |
||||||
|
return result; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
return {}; |
||||||
|
} |
||||||
@ -0,0 +1,20 @@ |
|||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <atomic> |
||||||
|
#include <string> |
||||||
|
|
||||||
|
class FileReader { |
||||||
|
public: |
||||||
|
FileReader(bool cache_to_local, int chunk_size = -1, int max_retries = 3) |
||||||
|
: cache_to_local_(cache_to_local), chunk_size_(chunk_size), max_retries_(max_retries) {} |
||||||
|
virtual ~FileReader() {} |
||||||
|
std::string read(const std::string &file, std::atomic<bool> *abort = nullptr); |
||||||
|
|
||||||
|
private: |
||||||
|
std::string download(const std::string &url, std::atomic<bool> *abort); |
||||||
|
int chunk_size_; |
||||||
|
int max_retries_; |
||||||
|
bool cache_to_local_; |
||||||
|
}; |
||||||
|
|
||||||
|
std::string cacheFilePath(const std::string &url); |
||||||
Loading…
Reference in new issue