replay: improve segment download and merge (#22654)
* no-cache mode
* fix test cases build error
* space
* don't create cache dir in no-cache mode
* fix errors in test cases
* no_local_cache_
* set the number of connections by chunk_size
* use size_t instead of int64_t
* add test case for no-cache mode
* rename variables
* fix SIGSEGV
* cleanup
* faster decompressBZ2
* always decompress bz2
* add test cases
* prepare for python interface
* fix test cases build error
* continue
* camera_replay: cache remote file
* protected inheritance
* single option name
* TODO
* test_case for LogReader&FrameReader
* fix wrong require
* test case for FileReader
* cleanup test
* test:fix wrong filename
* check cached file's checksum
* fix mkdir permissions err
cleanup filereader
* remove initialize libav network libraries.
dd
* abort all loading if one failed
* cleanup tests
* use threadpool to limit concurrent downloads
* cache more segments
* merge 3 segments for replay
* one segment uses about 100M of memory
* use segments_need_merge.size()
* shutdown
* fix stuck if exit replay before keyboard thread started
* load one segment at a time
* small cleanup
* cleanup filereader
* space
* tiny cleanup
* merge master
* cleanup test cases
* use util:create_directories
* cleanup framereader
old-commit-hash: 2b4a477fbc
commatwo_master
Dean Lee
4 years ago
committed by
GitHub
parent
3a8fdaba2f
commit
a031b938b0
19 changed files with 347 additions and 237 deletions
@ -0,0 +1,62 @@ |
||||
#include "selfdrive/ui/replay/filereader.h" |
||||
|
||||
#include <sys/stat.h> |
||||
|
||||
#include <cassert> |
||||
#include <cmath> |
||||
#include <fstream> |
||||
#include <iostream> |
||||
#include <sstream> |
||||
|
||||
#include "selfdrive/common/util.h" |
||||
#include "selfdrive/ui/replay/util.h" |
||||
|
||||
std::string cacheFilePath(const std::string &url) { |
||||
static std::string cache_path = [] { |
||||
const std::string comma_cache = util::getenv("COMMA_CACHE", "/tmp/comma_download_cache/"); |
||||
util::create_directories(comma_cache, 0755); |
||||
return comma_cache.back() == '/' ? comma_cache : comma_cache + "/"; |
||||
}(); |
||||
|
||||
return cache_path + sha256(getUrlWithoutQuery(url));; |
||||
} |
||||
|
||||
std::string FileReader::read(const std::string &file, std::atomic<bool> *abort) { |
||||
const bool is_remote = file.find("https://") == 0; |
||||
const std::string local_file = is_remote ? cacheFilePath(file) : file; |
||||
std::string result; |
||||
|
||||
if ((!is_remote || cache_to_local_) && util::file_exists(local_file)) { |
||||
result = util::read_file(local_file); |
||||
} else if (is_remote) { |
||||
result = download(file, abort); |
||||
if (cache_to_local_ && !result.empty()) { |
||||
std::ofstream fs(local_file, fs.binary | fs.out); |
||||
fs.write(result.data(), result.size()); |
||||
} |
||||
} |
||||
return result; |
||||
} |
||||
|
||||
std::string FileReader::download(const std::string &url, std::atomic<bool> *abort) { |
||||
std::string result; |
||||
size_t remote_file_size = 0; |
||||
for (int i = 0; i <= max_retries_ && !(abort && *abort); ++i) { |
||||
if (i > 0) { |
||||
std::cout << "download failed, retrying" << i << std::endl; |
||||
} |
||||
if (remote_file_size <= 0) { |
||||
remote_file_size = getRemoteFileSize(url); |
||||
} |
||||
if (remote_file_size > 0 && !(abort && *abort)) { |
||||
std::ostringstream oss; |
||||
result.resize(remote_file_size); |
||||
oss.rdbuf()->pubsetbuf(result.data(), result.size()); |
||||
int chunks = chunk_size_ > 0 ? std::min(1, (int)std::nearbyint(remote_file_size / (float)chunk_size_)) : 1; |
||||
if (httpMultiPartDownload(url, oss, chunks, remote_file_size, abort)) { |
||||
return result; |
||||
} |
||||
} |
||||
} |
||||
return {}; |
||||
} |
||||
@ -0,0 +1,20 @@ |
||||
#pragma once |
||||
|
||||
#include <atomic> |
||||
#include <string> |
||||
|
||||
class FileReader { |
||||
public: |
||||
FileReader(bool cache_to_local, int chunk_size = -1, int max_retries = 3) |
||||
: cache_to_local_(cache_to_local), chunk_size_(chunk_size), max_retries_(max_retries) {} |
||||
virtual ~FileReader() {} |
||||
std::string read(const std::string &file, std::atomic<bool> *abort = nullptr); |
||||
|
||||
private: |
||||
std::string download(const std::string &url, std::atomic<bool> *abort); |
||||
int chunk_size_; |
||||
int max_retries_; |
||||
bool cache_to_local_; |
||||
}; |
||||
|
||||
std::string cacheFilePath(const std::string &url); |
||||
Loading…
Reference in new issue