add zstd Python library (#32731)

* add zstd

* fix

* uploader

* logreader: zst support

* caps

* only logreader
old-commit-hash: 04a93dd849

poetry.lock

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6181d338c8da4314745ebb305485a8c22130288f3faa9740fb1782288c06bc0b
+oid sha256:35ddb7f95b0443e48f540d6b7ec0b01b4d35da2c7b758de47fb6f0040dac527d
-size 614543
+size 627912

pyproject.toml

@@ -187,6 +187,7 @@ tabulate = "*"
 types-requests = "*"
 types-tabulate = "*"
 tqdm = "*"
+zstd = "*"
 
 # this is only pinned since 5.15.11 is broken
 pyqt5 = { version = "==5.15.2", markers = "platform_machine == 'x86_64'" } # no aarch64 wheels for macOS/linux

tools/lib/logreader.py

@@ -10,6 +10,7 @@ import sys
 import tqdm
 import urllib.parse
 import warnings
+import zstd
 
 from collections.abc import Callable, Iterable, Iterator
 from urllib.parse import parse_qs, urlparse
@@ -34,8 +35,8 @@ class _LogFileReader:
     ext = None
     if not dat:
       _, ext = os.path.splitext(urllib.parse.urlparse(fn).path)
-      if ext not in ('', '.bz2'):
+      if ext not in ('', '.bz2', '.zst'):
-        # old rlogs weren't bz2 compressed
+        # old rlogs weren't compressed
         raise Exception(f"unknown extension {ext}")
 
       with FileReader(fn) as f:
@@ -43,6 +44,9 @@ class _LogFileReader:
 
     if ext == ".bz2" or dat.startswith(b'BZh9'):
       dat = bz2.decompress(dat)
+    elif ext == ".zst" or dat.startswith(b'\x28\xB5\x2F\xFD'):
+      # https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#zstandard-frames
+      dat = zstd.decompress(dat)
 
     ents = capnp_log.Event.read_multiple_bytes(dat)