qlog_size.py: use zstd (#33137)

qlog_size: use zstd
pull/33138/head
Shane Smiskol 9 months ago committed by GitHub
parent 84cff4fc03
commit 11cb2d3a0b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 7
      selfdrive/debug/internal/qlog_size.py

@ -1,11 +1,12 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import argparse import argparse
import bz2 import zstandard as zstd
from collections import defaultdict from collections import defaultdict
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
from cereal.services import SERVICE_LIST from cereal.services import SERVICE_LIST
from openpilot.system.loggerd.uploader import LOG_COMPRESSION_LEVEL
from openpilot.tools.lib.logreader import LogReader from openpilot.tools.lib.logreader import LogReader
from tqdm import tqdm from tqdm import tqdm
@ -17,14 +18,14 @@ def make_pie(msgs, typ):
for m in msgs: for m in msgs:
msgs_by_type[m.which()].append(m.as_builder().to_bytes()) msgs_by_type[m.which()].append(m.as_builder().to_bytes())
total = len(bz2.compress(b"".join([m.as_builder().to_bytes() for m in msgs]))) total = len(zstd.compress(b"".join([m.as_builder().to_bytes() for m in msgs]), LOG_COMPRESSION_LEVEL))
uncompressed_total = len(b"".join([m.as_builder().to_bytes() for m in msgs])) uncompressed_total = len(b"".join([m.as_builder().to_bytes() for m in msgs]))
length_by_type = {k: len(b"".join(v)) for k, v in msgs_by_type.items()} length_by_type = {k: len(b"".join(v)) for k, v in msgs_by_type.items()}
# calculate compressed size by calculating diff when removed from the segment # calculate compressed size by calculating diff when removed from the segment
compressed_length_by_type = {} compressed_length_by_type = {}
for k in tqdm(msgs_by_type.keys(), desc="Compressing"): for k in tqdm(msgs_by_type.keys(), desc="Compressing"):
compressed_length_by_type[k] = total - len(bz2.compress(b"".join([m.as_builder().to_bytes() for m in msgs if m.which() != k]))) compressed_length_by_type[k] = total - len(zstd.compress(b"".join([m.as_builder().to_bytes() for m in msgs if m.which() != k]), LOG_COMPRESSION_LEVEL))
sizes = sorted(compressed_length_by_type.items(), key=lambda kv: kv[1]) sizes = sorted(compressed_length_by_type.items(), key=lambda kv: kv[1])

Loading…
Cancel
Save