replay: improve segment download and merge (#22654)
* no-cache mode
* fix test cases build error
* space
* don't create cache dir in no-cache mode
* fix errors in test cases
* no_local_cache_
* set the number of connections by chunk_size
* use size_t instead of int64_t
* add test case for no-cache mode
* rename variables
* fix SIGSEGV
* cleanup
* faster decompressBZ2
* always decompress bz2
* add test cases
* prepare for python interface
* fix test cases build error
* continue
* camera_replay: cache remote file
* protected inheritance
* single option name
* TODO
* test_case for LogReader&FrameReader
* fix wrong require
* test case for FileReader
* cleanup test
* test:fix wrong filename
* check cached file's checksum
* fix mkdir permissions err
cleanup filereader
* remove initialize libav network libraries.
dd
* abort all loading if one failed
* cleanup tests
* use threadpool to limit concurrent downloads
* cache more segments
* merge 3 segments for replay
* one segment uses about 100M of memory
* use segments_need_merge.size()
* shutdown
* fix stuck if exit replay before keyboard thread started
* load one segment at a time
* small cleanup
* cleanup filereader
* space
* tiny cleanup
* merge master
* cleanup test cases
* use util:create_directories
* cleanup framereader
old-commit-hash: 2b4a477fbc
commatwo_master
Dean Lee
4 years ago
committed by
GitHub
parent
3a8fdaba2f
commit
a031b938b0
19 changed files with 347 additions and 237 deletions
@ -0,0 +1,62 @@ |
|||||||
|
#include "selfdrive/ui/replay/filereader.h" |
||||||
|
|
||||||
|
#include <sys/stat.h> |
||||||
|
|
||||||
|
#include <cassert> |
||||||
|
#include <cmath> |
||||||
|
#include <fstream> |
||||||
|
#include <iostream> |
||||||
|
#include <sstream> |
||||||
|
|
||||||
|
#include "selfdrive/common/util.h" |
||||||
|
#include "selfdrive/ui/replay/util.h" |
||||||
|
|
||||||
|
std::string cacheFilePath(const std::string &url) { |
||||||
|
static std::string cache_path = [] { |
||||||
|
const std::string comma_cache = util::getenv("COMMA_CACHE", "/tmp/comma_download_cache/"); |
||||||
|
util::create_directories(comma_cache, 0755); |
||||||
|
return comma_cache.back() == '/' ? comma_cache : comma_cache + "/"; |
||||||
|
}(); |
||||||
|
|
||||||
|
return cache_path + sha256(getUrlWithoutQuery(url));; |
||||||
|
} |
||||||
|
|
||||||
|
std::string FileReader::read(const std::string &file, std::atomic<bool> *abort) { |
||||||
|
const bool is_remote = file.find("https://") == 0; |
||||||
|
const std::string local_file = is_remote ? cacheFilePath(file) : file; |
||||||
|
std::string result; |
||||||
|
|
||||||
|
if ((!is_remote || cache_to_local_) && util::file_exists(local_file)) { |
||||||
|
result = util::read_file(local_file); |
||||||
|
} else if (is_remote) { |
||||||
|
result = download(file, abort); |
||||||
|
if (cache_to_local_ && !result.empty()) { |
||||||
|
std::ofstream fs(local_file, fs.binary | fs.out); |
||||||
|
fs.write(result.data(), result.size()); |
||||||
|
} |
||||||
|
} |
||||||
|
return result; |
||||||
|
} |
||||||
|
|
||||||
|
std::string FileReader::download(const std::string &url, std::atomic<bool> *abort) { |
||||||
|
std::string result; |
||||||
|
size_t remote_file_size = 0; |
||||||
|
for (int i = 0; i <= max_retries_ && !(abort && *abort); ++i) { |
||||||
|
if (i > 0) { |
||||||
|
std::cout << "download failed, retrying" << i << std::endl; |
||||||
|
} |
||||||
|
if (remote_file_size <= 0) { |
||||||
|
remote_file_size = getRemoteFileSize(url); |
||||||
|
} |
||||||
|
if (remote_file_size > 0 && !(abort && *abort)) { |
||||||
|
std::ostringstream oss; |
||||||
|
result.resize(remote_file_size); |
||||||
|
oss.rdbuf()->pubsetbuf(result.data(), result.size()); |
||||||
|
int chunks = chunk_size_ > 0 ? std::min(1, (int)std::nearbyint(remote_file_size / (float)chunk_size_)) : 1; |
||||||
|
if (httpMultiPartDownload(url, oss, chunks, remote_file_size, abort)) { |
||||||
|
return result; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
return {}; |
||||||
|
} |
||||||
@ -0,0 +1,20 @@ |
|||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <atomic> |
||||||
|
#include <string> |
||||||
|
|
||||||
|
class FileReader { |
||||||
|
public: |
||||||
|
FileReader(bool cache_to_local, int chunk_size = -1, int max_retries = 3) |
||||||
|
: cache_to_local_(cache_to_local), chunk_size_(chunk_size), max_retries_(max_retries) {} |
||||||
|
virtual ~FileReader() {} |
||||||
|
std::string read(const std::string &file, std::atomic<bool> *abort = nullptr); |
||||||
|
|
||||||
|
private: |
||||||
|
std::string download(const std::string &url, std::atomic<bool> *abort); |
||||||
|
int chunk_size_; |
||||||
|
int max_retries_; |
||||||
|
bool cache_to_local_; |
||||||
|
}; |
||||||
|
|
||||||
|
std::string cacheFilePath(const std::string &url); |
||||||
Loading…
Reference in new issue