loggerd: switch logging from raw file format to zstd compressed files (#34549)
* switch logging from raw file format to zstd compressed files * more zst suffix * compress bootlog * remove class RawFile * Optimize ZstdFileWriter by adding input caching * use ZSTD_compressStream2 * cleanup * LOG_COMPRESSION_LEVEL=10 * space * add zst suffix to LOGS_SIZE_RATEpull/34551/head
Dean Lee
3 months ago
committed by
GitHub
parent
4066d49d70
commit
9ec54f59c6
14 changed files with 200 additions and 51 deletions
@ -0,0 +1,38 @@ |
|||||||
|
#include <zstd.h> |
||||||
|
|
||||||
|
#include <catch2/catch.hpp> |
||||||
|
#include <cstring> |
||||||
|
#include <vector> |
||||||
|
|
||||||
|
#include "common/util.h" |
||||||
|
#include "system/loggerd/logger.h" |
||||||
|
#include "system/loggerd/zstd_writer.h" |
||||||
|
|
||||||
|
TEST_CASE("ZstdFileWriter writes and compresses data correctly in loops", "[ZstdFileWriter]") { |
||||||
|
const std::string filename = "test_zstd_file.zst"; |
||||||
|
const int iterations = 100; |
||||||
|
const size_t dataSize = 1024; |
||||||
|
|
||||||
|
std::string totalTestData; |
||||||
|
|
||||||
|
// Step 1: Write compressed data to file in a loop
|
||||||
|
{ |
||||||
|
ZstdFileWriter writer(filename, LOG_COMPRESSION_LEVEL); |
||||||
|
for (int i = 0; i < iterations; ++i) { |
||||||
|
std::string testData = util::random_string(dataSize); |
||||||
|
totalTestData.append(testData); |
||||||
|
writer.write((void *)testData.c_str(), testData.size()); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Step 2: Decompress the file and verify the data
|
||||||
|
auto compressedContent = util::read_file(filename); |
||||||
|
std::string decompressedData = zstd_decompress(compressedContent); |
||||||
|
|
||||||
|
// Step 3: Verify that the decompressed data matches the original accumulated data
|
||||||
|
REQUIRE(decompressedData.size() == totalTestData.size()); |
||||||
|
REQUIRE(std::memcmp(decompressedData.data(), totalTestData.c_str(), totalTestData.size()) == 0); |
||||||
|
|
||||||
|
// Clean up the test file
|
||||||
|
std::remove(filename.c_str()); |
||||||
|
} |
||||||
@ -0,0 +1,65 @@ |
|||||||
|
|
||||||
|
#include "system/loggerd/zstd_writer.h" |
||||||
|
|
||||||
|
#include <cassert> |
||||||
|
|
||||||
|
#include "common/util.h" |
||||||
|
|
||||||
|
// Constructor: Initializes compression stream and opens file
|
||||||
|
ZstdFileWriter::ZstdFileWriter(const std::string& filename, int compression_level) { |
||||||
|
// Create the compression stream
|
||||||
|
cstream_ = ZSTD_createCStream(); |
||||||
|
assert(cstream_); |
||||||
|
|
||||||
|
size_t initResult = ZSTD_initCStream(cstream_, compression_level); |
||||||
|
assert(!ZSTD_isError(initResult)); |
||||||
|
|
||||||
|
input_cache_capacity_ = ZSTD_CStreamInSize(); |
||||||
|
input_cache_.reserve(input_cache_capacity_); |
||||||
|
output_buffer_.resize(ZSTD_CStreamOutSize()); |
||||||
|
|
||||||
|
file_ = util::safe_fopen(filename.c_str(), "wb"); |
||||||
|
assert(file_ != nullptr); |
||||||
|
} |
||||||
|
|
||||||
|
// Destructor: Finalizes compression and closes file
|
||||||
|
ZstdFileWriter::~ZstdFileWriter() { |
||||||
|
flushCache(true); |
||||||
|
util::safe_fflush(file_); |
||||||
|
|
||||||
|
int err = fclose(file_); |
||||||
|
assert(err == 0); |
||||||
|
|
||||||
|
ZSTD_freeCStream(cstream_); |
||||||
|
} |
||||||
|
|
||||||
|
// Compresses and writes data to file
|
||||||
|
void ZstdFileWriter::write(void* data, size_t size) { |
||||||
|
// Add data to the input cache
|
||||||
|
input_cache_.insert(input_cache_.end(), (uint8_t*)data, (uint8_t*)data + size); |
||||||
|
|
||||||
|
// If the cache is full, compress and write to the file
|
||||||
|
if (input_cache_.size() >= input_cache_capacity_) { |
||||||
|
flushCache(false); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Compress and flush the input cache to the file
|
||||||
|
void ZstdFileWriter::flushCache(bool last_chunk) { |
||||||
|
ZSTD_inBuffer input = {input_cache_.data(), input_cache_.size(), 0}; |
||||||
|
ZSTD_EndDirective mode = !last_chunk ? ZSTD_e_continue : ZSTD_e_end; |
||||||
|
int finished = 0; |
||||||
|
|
||||||
|
do { |
||||||
|
ZSTD_outBuffer output = {output_buffer_.data(), output_buffer_.size(), 0}; |
||||||
|
size_t remaining = ZSTD_compressStream2(cstream_, &output, &input, mode); |
||||||
|
assert(!ZSTD_isError(remaining)); |
||||||
|
|
||||||
|
size_t written = util::safe_fwrite(output_buffer_.data(), 1, output.pos, file_); |
||||||
|
assert(written == output.pos); |
||||||
|
|
||||||
|
finished = last_chunk ? (remaining == 0) : (input.pos == input.size); |
||||||
|
} while (!finished); |
||||||
|
|
||||||
|
input_cache_.clear(); // Clear cache after compression
|
||||||
|
} |
||||||
@ -0,0 +1,24 @@ |
|||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <zstd.h> |
||||||
|
|
||||||
|
#include <string> |
||||||
|
#include <vector> |
||||||
|
#include <capnp/common.h> |
||||||
|
|
||||||
|
class ZstdFileWriter { |
||||||
|
public: |
||||||
|
ZstdFileWriter(const std::string &filename, int compression_level); |
||||||
|
~ZstdFileWriter(); |
||||||
|
void write(void* data, size_t size); |
||||||
|
inline void write(kj::ArrayPtr<capnp::byte> array) { write(array.begin(), array.size()); } |
||||||
|
|
||||||
|
private: |
||||||
|
void flushCache(bool last_chunk); |
||||||
|
|
||||||
|
size_t input_cache_capacity_ = 0; |
||||||
|
std::vector<char> input_cache_; |
||||||
|
std::vector<char> output_buffer_; |
||||||
|
ZSTD_CStream *cstream_; |
||||||
|
FILE* file_ = nullptr; |
||||||
|
}; |
||||||
Loading…
Reference in new issue