loggerd: switch logging from raw file format to zstd compressed files (#34549)
* switch logging from raw file format to zstd compressed files * more zst suffix * compress bootlog * remove class RawFile * Optimize ZstdFileWriter by adding input caching * use ZSTD_compressStream2 * cleanup * LOG_COMPRESSION_LEVEL=10 * space * add zst suffix to LOGS_SIZE_RATEpull/34551/head
Dean Lee
3 months ago
committed by
GitHub
parent
4066d49d70
commit
9ec54f59c6
14 changed files with 200 additions and 51 deletions
@ -0,0 +1,38 @@ |
||||
#include <zstd.h> |
||||
|
||||
#include <catch2/catch.hpp> |
||||
#include <cstring> |
||||
#include <vector> |
||||
|
||||
#include "common/util.h" |
||||
#include "system/loggerd/logger.h" |
||||
#include "system/loggerd/zstd_writer.h" |
||||
|
||||
TEST_CASE("ZstdFileWriter writes and compresses data correctly in loops", "[ZstdFileWriter]") { |
||||
const std::string filename = "test_zstd_file.zst"; |
||||
const int iterations = 100; |
||||
const size_t dataSize = 1024; |
||||
|
||||
std::string totalTestData; |
||||
|
||||
// Step 1: Write compressed data to file in a loop
|
||||
{ |
||||
ZstdFileWriter writer(filename, LOG_COMPRESSION_LEVEL); |
||||
for (int i = 0; i < iterations; ++i) { |
||||
std::string testData = util::random_string(dataSize); |
||||
totalTestData.append(testData); |
||||
writer.write((void *)testData.c_str(), testData.size()); |
||||
} |
||||
} |
||||
|
||||
// Step 2: Decompress the file and verify the data
|
||||
auto compressedContent = util::read_file(filename); |
||||
std::string decompressedData = zstd_decompress(compressedContent); |
||||
|
||||
// Step 3: Verify that the decompressed data matches the original accumulated data
|
||||
REQUIRE(decompressedData.size() == totalTestData.size()); |
||||
REQUIRE(std::memcmp(decompressedData.data(), totalTestData.c_str(), totalTestData.size()) == 0); |
||||
|
||||
// Clean up the test file
|
||||
std::remove(filename.c_str()); |
||||
} |
||||
@ -0,0 +1,65 @@ |
||||
|
||||
#include "system/loggerd/zstd_writer.h" |
||||
|
||||
#include <cassert> |
||||
|
||||
#include "common/util.h" |
||||
|
||||
// Constructor: Initializes compression stream and opens file
|
||||
ZstdFileWriter::ZstdFileWriter(const std::string& filename, int compression_level) { |
||||
// Create the compression stream
|
||||
cstream_ = ZSTD_createCStream(); |
||||
assert(cstream_); |
||||
|
||||
size_t initResult = ZSTD_initCStream(cstream_, compression_level); |
||||
assert(!ZSTD_isError(initResult)); |
||||
|
||||
input_cache_capacity_ = ZSTD_CStreamInSize(); |
||||
input_cache_.reserve(input_cache_capacity_); |
||||
output_buffer_.resize(ZSTD_CStreamOutSize()); |
||||
|
||||
file_ = util::safe_fopen(filename.c_str(), "wb"); |
||||
assert(file_ != nullptr); |
||||
} |
||||
|
||||
// Destructor: Finalizes compression and closes file
|
||||
ZstdFileWriter::~ZstdFileWriter() { |
||||
flushCache(true); |
||||
util::safe_fflush(file_); |
||||
|
||||
int err = fclose(file_); |
||||
assert(err == 0); |
||||
|
||||
ZSTD_freeCStream(cstream_); |
||||
} |
||||
|
||||
// Compresses and writes data to file
|
||||
void ZstdFileWriter::write(void* data, size_t size) { |
||||
// Add data to the input cache
|
||||
input_cache_.insert(input_cache_.end(), (uint8_t*)data, (uint8_t*)data + size); |
||||
|
||||
// If the cache is full, compress and write to the file
|
||||
if (input_cache_.size() >= input_cache_capacity_) { |
||||
flushCache(false); |
||||
} |
||||
} |
||||
|
||||
// Compress and flush the input cache to the file
|
||||
void ZstdFileWriter::flushCache(bool last_chunk) { |
||||
ZSTD_inBuffer input = {input_cache_.data(), input_cache_.size(), 0}; |
||||
ZSTD_EndDirective mode = !last_chunk ? ZSTD_e_continue : ZSTD_e_end; |
||||
int finished = 0; |
||||
|
||||
do { |
||||
ZSTD_outBuffer output = {output_buffer_.data(), output_buffer_.size(), 0}; |
||||
size_t remaining = ZSTD_compressStream2(cstream_, &output, &input, mode); |
||||
assert(!ZSTD_isError(remaining)); |
||||
|
||||
size_t written = util::safe_fwrite(output_buffer_.data(), 1, output.pos, file_); |
||||
assert(written == output.pos); |
||||
|
||||
finished = last_chunk ? (remaining == 0) : (input.pos == input.size); |
||||
} while (!finished); |
||||
|
||||
input_cache_.clear(); // Clear cache after compression
|
||||
} |
||||
@ -0,0 +1,24 @@ |
||||
#pragma once |
||||
|
||||
#include <zstd.h> |
||||
|
||||
#include <string> |
||||
#include <vector> |
||||
#include <capnp/common.h> |
||||
|
||||
class ZstdFileWriter { |
||||
public: |
||||
ZstdFileWriter(const std::string &filename, int compression_level); |
||||
~ZstdFileWriter(); |
||||
void write(void* data, size_t size); |
||||
inline void write(kj::ArrayPtr<capnp::byte> array) { write(array.begin(), array.size()); } |
||||
|
||||
private: |
||||
void flushCache(bool last_chunk); |
||||
|
||||
size_t input_cache_capacity_ = 0; |
||||
std::vector<char> input_cache_; |
||||
std::vector<char> output_buffer_; |
||||
ZSTD_CStream *cstream_; |
||||
FILE* file_ = nullptr; |
||||
}; |
||||
Loading…
Reference in new issue