diff --git a/common/dict_helpers.py b/common/dict_helpers.py deleted file mode 100644 index 62cff63b58..0000000000 --- a/common/dict_helpers.py +++ /dev/null @@ -1,9 +0,0 @@ -# remove all keys that end in DEPRECATED -def strip_deprecated_keys(d): - for k in list(d.keys()): - if isinstance(k, str): - if k.endswith('DEPRECATED'): - d.pop(k) - elif isinstance(d[k], dict): - strip_deprecated_keys(d[k]) - return d diff --git a/common/git.py b/common/git.py index 4406bf96b1..2296fa7088 100644 --- a/common/git.py +++ b/common/git.py @@ -1,6 +1,6 @@ from functools import cache import subprocess -from openpilot.common.run import run_cmd, run_cmd_default +from openpilot.common.utils import run_cmd, run_cmd_default @cache diff --git a/common/retry.py b/common/retry.py deleted file mode 100644 index 9bd4ac9522..0000000000 --- a/common/retry.py +++ /dev/null @@ -1,30 +0,0 @@ -import time -import functools - -from openpilot.common.swaglog import cloudlog - - -def retry(attempts=3, delay=1.0, ignore_failure=False): - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - for _ in range(attempts): - try: - return func(*args, **kwargs) - except Exception: - cloudlog.exception(f"{func.__name__} failed, trying again") - time.sleep(delay) - - if ignore_failure: - cloudlog.error(f"{func.__name__} failed after retry") - else: - raise Exception(f"{func.__name__} failed after retry") - return wrapper - return decorator - - -if __name__ == "__main__": - @retry(attempts=10) - def abc(): - raise ValueError("abc failed :(") - abc() diff --git a/common/run.py b/common/run.py deleted file mode 100644 index 75395ead1f..0000000000 --- a/common/run.py +++ /dev/null @@ -1,28 +0,0 @@ -import subprocess -from contextlib import contextmanager -from subprocess import Popen, PIPE, TimeoutExpired - - -def run_cmd(cmd: list[str], cwd=None, env=None) -> str: - return subprocess.check_output(cmd, encoding='utf8', cwd=cwd, env=env).strip() - - -def run_cmd_default(cmd: list[str], default: str = "", cwd=None, env=None) -> str: - try: - return run_cmd(cmd, cwd=cwd, env=env) - except subprocess.CalledProcessError: - return default - - -@contextmanager -def managed_proc(cmd: list[str], env: dict[str, str]): - proc = Popen(cmd, env=env, stdout=PIPE, stderr=PIPE) - try: - yield proc - finally: - if proc.poll() is None: - proc.terminate() - try: - proc.wait(timeout=5) - except TimeoutExpired: - proc.kill() diff --git a/common/tests/test_file_helpers.py b/common/tests/test_file_helpers.py index a9977c2362..c7fe1984c5 100644 --- a/common/tests/test_file_helpers.py +++ b/common/tests/test_file_helpers.py @@ -1,7 +1,7 @@ import os from uuid import uuid4 -from openpilot.common.file_helpers import atomic_write_in_dir +from openpilot.common.utils import atomic_write_in_dir class TestFileHelpers: diff --git a/common/file_helpers.py b/common/utils.py similarity index 51% rename from common/file_helpers.py rename to common/utils.py index b0d889f163..89c0601f06 100644 --- a/common/file_helpers.py +++ b/common/utils.py @@ -2,9 +2,14 @@ import io import os import tempfile import contextlib +import subprocess +import time +import functools +from subprocess import Popen, PIPE, TimeoutExpired import zstandard as zstd +from openpilot.common.swaglog import cloudlog -LOG_COMPRESSION_LEVEL = 10 # little benefit up to level 15. level ~17 is a small step change +LOG_COMPRESSION_LEVEL = 10 # little benefit up to level 15. level ~17 is a small step change class CallbackReader: @@ -27,7 +32,7 @@ class CallbackReader: @contextlib.contextmanager -def atomic_write_in_dir(path: str, mode: str = 'w', buffering: int = -1, encoding: str = None, newline: str = None, +def atomic_write_in_dir(path: str, mode: str = 'w', buffering: int = -1, encoding: str | None = None, newline: str | None = None, overwrite: bool = False): """Write to a file atomically using a temporary file in the same directory as the destination file.""" dir_name = os.path.dirname(path) @@ -56,3 +61,58 @@ def get_upload_stream(filepath: str, should_compress: bool) -> tuple[io.Buffered compressed_size = compressed_stream.tell() compressed_stream.seek(0) return compressed_stream, compressed_size + + +# remove all keys that end in DEPRECATED +def strip_deprecated_keys(d): + for k in list(d.keys()): + if isinstance(k, str): + if k.endswith('DEPRECATED'): + d.pop(k) + elif isinstance(d[k], dict): + strip_deprecated_keys(d[k]) + return d + + +def run_cmd(cmd: list[str], cwd=None, env=None) -> str: + return subprocess.check_output(cmd, encoding='utf8', cwd=cwd, env=env).strip() + + +def run_cmd_default(cmd: list[str], default: str = "", cwd=None, env=None) -> str: + try: + return run_cmd(cmd, cwd=cwd, env=env) + except subprocess.CalledProcessError: + return default + + +@contextlib.contextmanager +def managed_proc(cmd: list[str], env: dict[str, str]): + proc = Popen(cmd, env=env, stdout=PIPE, stderr=PIPE) + try: + yield proc + finally: + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=5) + except TimeoutExpired: + proc.kill() + + +def retry(attempts=3, delay=1.0, ignore_failure=False): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + for _ in range(attempts): + try: + return func(*args, **kwargs) + except Exception: + cloudlog.exception(f"{func.__name__} failed, trying again") + time.sleep(delay) + + if ignore_failure: + cloudlog.error(f"{func.__name__} failed after retry") + else: + raise Exception(f"{func.__name__} failed after retry") + return wrapper + return decorator diff --git a/selfdrive/debug/qlog_size.py b/selfdrive/debug/qlog_size.py index 6d494b6f75..2b54cfeebf 100755 --- a/selfdrive/debug/qlog_size.py +++ b/selfdrive/debug/qlog_size.py @@ -6,7 +6,7 @@ from collections import defaultdict import matplotlib.pyplot as plt from cereal.services import SERVICE_LIST -from openpilot.common.file_helpers import LOG_COMPRESSION_LEVEL +from openpilot.common.utils import LOG_COMPRESSION_LEVEL from openpilot.tools.lib.logreader import LogReader from tqdm import tqdm diff --git a/selfdrive/pandad/tests/test_pandad_loopback.py b/selfdrive/pandad/tests/test_pandad_loopback.py index bf1c557128..eff70d2544 100644 --- a/selfdrive/pandad/tests/test_pandad_loopback.py +++ b/selfdrive/pandad/tests/test_pandad_loopback.py @@ -9,7 +9,7 @@ from pprint import pprint import cereal.messaging as messaging from cereal import car, log from opendbc.car.can_definitions import CanData -from openpilot.common.retry import retry +from openpilot.common.utils import retry from openpilot.common.params import Params from openpilot.common.timeout import Timeout from openpilot.selfdrive.pandad import can_list_to_can_capnp diff --git a/selfdrive/ui/soundd.py b/selfdrive/ui/soundd.py index 44c463b18c..44116a2329 100644 --- a/selfdrive/ui/soundd.py +++ b/selfdrive/ui/soundd.py @@ -8,7 +8,7 @@ from cereal import car, messaging from openpilot.common.basedir import BASEDIR from openpilot.common.filter_simple import FirstOrderFilter from openpilot.common.realtime import Ratekeeper -from openpilot.common.retry import retry +from openpilot.common.utils import retry from openpilot.common.swaglog import cloudlog from openpilot.system import micd diff --git a/system/athena/athenad.py b/system/athena/athenad.py index 6ed53b759c..50c4f5408f 100755 --- a/system/athena/athenad.py +++ b/system/athena/athenad.py @@ -31,7 +31,7 @@ import cereal.messaging as messaging from cereal import log from cereal.services import SERVICE_LIST from openpilot.common.api import Api -from openpilot.common.file_helpers import CallbackReader, get_upload_stream +from openpilot.common.utils import CallbackReader, get_upload_stream from openpilot.common.params import Params from openpilot.common.realtime import set_core_affinity from openpilot.system.hardware import HARDWARE, PC diff --git a/system/hardware/hardwared.py b/system/hardware/hardwared.py index 1048acfe0a..1dce99273a 100755 --- a/system/hardware/hardwared.py +++ b/system/hardware/hardwared.py @@ -12,7 +12,7 @@ import psutil import cereal.messaging as messaging from cereal import log from cereal.services import SERVICE_LIST -from openpilot.common.dict_helpers import strip_deprecated_keys +from openpilot.common.utils import strip_deprecated_keys from openpilot.common.filter_simple import FirstOrderFilter from openpilot.common.params import Params from openpilot.common.realtime import DT_HW diff --git a/system/loggerd/uploader.py b/system/loggerd/uploader.py index bc19572507..5b6234e1d5 100755 --- a/system/loggerd/uploader.py +++ b/system/loggerd/uploader.py @@ -12,7 +12,7 @@ from collections.abc import Iterator from cereal import log import cereal.messaging as messaging from openpilot.common.api import Api -from openpilot.common.file_helpers import get_upload_stream +from openpilot.common.utils import get_upload_stream from openpilot.common.params import Params from openpilot.common.realtime import set_core_affinity from openpilot.system.hardware.hw import Paths diff --git a/system/micd.py b/system/micd.py index b3558a15a8..9b3ccc8d29 100755 --- a/system/micd.py +++ b/system/micd.py @@ -5,7 +5,7 @@ import threading from cereal import messaging from openpilot.common.realtime import Ratekeeper -from openpilot.common.retry import retry +from openpilot.common.utils import retry from openpilot.common.swaglog import cloudlog RATE = 10 diff --git a/system/qcomgpsd/qcomgpsd.py b/system/qcomgpsd/qcomgpsd.py index 819b17f113..59f5ac0b50 100755 --- a/system/qcomgpsd/qcomgpsd.py +++ b/system/qcomgpsd/qcomgpsd.py @@ -16,7 +16,7 @@ from struct import unpack_from, calcsize, pack from cereal import log import cereal.messaging as messaging from openpilot.common.gpio import gpio_init, gpio_set -from openpilot.common.retry import retry +from openpilot.common.utils import retry from openpilot.common.time_helpers import system_time_valid from openpilot.system.hardware.tici.pins import GPIO from openpilot.common.swaglog import cloudlog diff --git a/system/statsd.py b/system/statsd.py index d60064fc91..c4216f5e76 100755 --- a/system/statsd.py +++ b/system/statsd.py @@ -13,7 +13,7 @@ from cereal.messaging import SubMaster from openpilot.system.hardware.hw import Paths from openpilot.common.swaglog import cloudlog from openpilot.system.hardware import HARDWARE -from openpilot.common.file_helpers import atomic_write_in_dir +from openpilot.common.utils import atomic_write_in_dir from openpilot.system.version import get_build_metadata from openpilot.system.loggerd.config import STATS_DIR_FILE_LIMIT, STATS_SOCKET, STATS_FLUSH_TIME_S diff --git a/system/ui/setup.py b/system/ui/setup.py index 5dbf597484..da8c8d81fb 100755 --- a/system/ui/setup.py +++ b/system/ui/setup.py @@ -11,7 +11,7 @@ import shutil import pyray as rl from cereal import log -from openpilot.common.run import run_cmd +from openpilot.common.utils import run_cmd from openpilot.system.hardware import HARDWARE from openpilot.system.ui.lib.scroll_panel import GuiScrollPanel from openpilot.system.ui.lib.application import gui_app, FontWeight, FONT_SCALE diff --git a/tools/clip/run.py b/tools/clip/run.py index 8fa0e8eda3..9045a4381b 100755 --- a/tools/clip/run.py +++ b/tools/clip/run.py @@ -17,7 +17,7 @@ from cereal.messaging import SubMaster from openpilot.common.basedir import BASEDIR from openpilot.common.params import Params, UnknownKeyName from openpilot.common.prefix import OpenpilotPrefix -from openpilot.common.run import managed_proc +from openpilot.common.utils import managed_proc from openpilot.tools.lib.route import Route from openpilot.tools.lib.logreader import LogReader diff --git a/tools/lib/filereader.py b/tools/lib/filereader.py index 02f5fd1b95..ee9ee294bb 100644 --- a/tools/lib/filereader.py +++ b/tools/lib/filereader.py @@ -2,7 +2,7 @@ import os import posixpath import socket from functools import cache -from openpilot.common.retry import retry +from openpilot.common.utils import retry from urllib.parse import urlparse from openpilot.tools.lib.url_file import URLFile diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index e80ba1399d..31c1e0ff11 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -7,7 +7,7 @@ from urllib3 import PoolManager, Retry from urllib3.response import BaseHTTPResponse from urllib3.util import Timeout -from openpilot.common.file_helpers import atomic_write_in_dir +from openpilot.common.utils import atomic_write_in_dir from openpilot.system.hardware.hw import Paths # Cache chunk size