From 6005b12f940b96f14ccf87fb14c37a9c2fca8948 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 19 Aug 2025 15:04:17 -0700 Subject: [PATCH] format logreader --- tools/lib/logreader.py | 2 ++ tools/lib/route.py | 7 +++---- tools/lib/url_file.py | 18 ++++++++++-------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index bedafb8566..833f9825bb 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -40,6 +40,7 @@ def save_log(dest, log_msgs, compress=True): with open(dest, "wb") as f: f.write(dat) + def decompress_stream(data: bytes): dctx = zstd.ZstdDecompressor() decompressed_data = b"" @@ -353,6 +354,7 @@ class LogReader: def time_series(self): return msgs_to_time_series(self) + if __name__ == "__main__": import codecs diff --git a/tools/lib/route.py b/tools/lib/route.py index 882585a151..1fc26fb996 100644 --- a/tools/lib/route.py +++ b/tools/lib/route.py @@ -231,7 +231,6 @@ class RouteName: def __str__(self) -> str: return self._canonical_name - class SegmentName: # TODO: add constructor that takes dongle_id, time_str, segment_num and then create instances # of this class instead of manually constructing a segment name (use canonical_name prop instead) @@ -252,7 +251,7 @@ class SegmentName: @property def canonical_name(self) -> str: return self._canonical_name - #TODO should only use one name + # TODO should only use one name @property def data_name(self) -> str: return f"{self._route_name.canonical_name}/{self._num}" @@ -283,7 +282,7 @@ class SegmentName: @staticmethod def from_file_name(file_name): # ??????/xxxxxxxxxxxxxxxx|1111-11-11-11--11-11-11/1/rlog.bz2 - dongle_id, route_name, segment_num = file_name.replace('|','/').split('/')[-4:-1] + dongle_id, route_name, segment_num = file_name.replace('|', '/').split('/')[-4:-1] return SegmentName(dongle_id + "|" + route_name + "--" + segment_num) @staticmethod @@ -304,6 +303,7 @@ class SegmentName: dongle_id, route_name, segment_num = prefix.split("/") return SegmentName(dongle_id + "|" + route_name + "--" + segment_num) + @cache def get_max_seg_number_cached(sr: 'SegmentRange') -> int: try: @@ -365,4 +365,3 @@ class SegmentRange: def __repr__(self) -> str: return self.__str__() - diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index 204726363d..e80ba1399d 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -9,12 +9,14 @@ from urllib3.util import Timeout from openpilot.common.file_helpers import atomic_write_in_dir from openpilot.system.hardware.hw import Paths + # Cache chunk size K = 1000 CHUNK_SIZE = 1000 * K logging.getLogger("urllib3").setLevel(logging.WARNING) + def hash_256(link: str) -> str: return sha256((link.split("?")[0]).encode('utf-8')).hexdigest() @@ -24,7 +26,7 @@ class URLFileException(Exception): class URLFile: - _pool_manager: PoolManager|None = None + _pool_manager: PoolManager | None = None @staticmethod def reset() -> None: @@ -33,16 +35,16 @@ class URLFile: @staticmethod def pool_manager() -> PoolManager: if URLFile._pool_manager is None: - socket_options = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),] + socket_options = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)] retries = Retry(total=5, backoff_factor=0.5, status_forcelist=[409, 429, 503, 504]) URLFile._pool_manager = PoolManager(num_pools=10, maxsize=100, socket_options=socket_options, retries=retries) return URLFile._pool_manager - def __init__(self, url: str, timeout: int=10, debug: bool=False, cache: bool|None=None): + def __init__(self, url: str, timeout: int = 10, debug: bool = False, cache: bool | None = None): self._url = url self._timeout = Timeout(connect=timeout, read=timeout) self._pos = 0 - self._length: int|None = None + self._length: int | None = None self._debug = debug # True by default, false if FILEREADER_CACHE is defined, but can be overwritten by the cache input self._force_download = not int(os.environ.get("FILEREADER_CACHE", "0")) @@ -58,7 +60,7 @@ class URLFile: def __exit__(self, exc_type, exc_value, traceback) -> None: pass - def _request(self, method: str, url: str, headers: dict[str, str]|None=None) -> BaseHTTPResponse: + def _request(self, method: str, url: str, headers: dict[str, str] | None = None) -> BaseHTTPResponse: return URLFile.pool_manager().request(method, url, timeout=self._timeout, headers=headers) def get_length_online(self) -> int: @@ -85,7 +87,7 @@ class URLFile: file_length.write(str(self._length)) return self._length - def read(self, ll: int|None=None) -> bytes: + def read(self, ll: int | None = None) -> bytes: if self._force_download: return self.read_aux(ll=ll) @@ -117,7 +119,7 @@ class URLFile: self._pos = file_end return response - def read_aux(self, ll: int|None=None) -> bytes: + def read_aux(self, ll: int | None = None) -> bytes: download_range = False headers = {} if self._pos != 0 or ll is not None: @@ -152,7 +154,7 @@ class URLFile: self._pos += len(ret) return ret - def seek(self, pos:int) -> None: + def seek(self, pos: int) -> None: self._pos = pos @property