merge a bunch of misc stuff into common.utils (#36463)

just utils

common/dict_helpers.py

@@ -1,9 +0,0 @@
-# remove all keys that end in DEPRECATED
-def strip_deprecated_keys(d):
-  for k in list(d.keys()):
-    if isinstance(k, str):
-      if k.endswith('DEPRECATED'):
-        d.pop(k)
-      elif isinstance(d[k], dict):
-        strip_deprecated_keys(d[k])
-  return d

common/git.py

@@ -1,6 +1,6 @@
 from functools import cache
 import subprocess
-from openpilot.common.run import run_cmd, run_cmd_default
+from openpilot.common.utils import run_cmd, run_cmd_default
 
 
 @cache

common/retry.py

@@ -1,30 +0,0 @@
-import time
-import functools
-
-from openpilot.common.swaglog import cloudlog
-
-
-def retry(attempts=3, delay=1.0, ignore_failure=False):
-  def decorator(func):
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-      for _ in range(attempts):
-        try:
-          return func(*args, **kwargs)
-        except Exception:
-          cloudlog.exception(f"{func.__name__} failed, trying again")
-          time.sleep(delay)
-
-      if ignore_failure:
-        cloudlog.error(f"{func.__name__} failed after retry")
-      else:
-        raise Exception(f"{func.__name__} failed after retry")
-    return wrapper
-  return decorator
-
-
-if __name__ == "__main__":
-  @retry(attempts=10)
-  def abc():
-    raise ValueError("abc failed :(")
-  abc()

common/run.py

@@ -1,28 +0,0 @@
-import subprocess
-from contextlib import contextmanager
-from subprocess import Popen, PIPE, TimeoutExpired
-
-
-def run_cmd(cmd: list[str], cwd=None, env=None) -> str:
-  return subprocess.check_output(cmd, encoding='utf8', cwd=cwd, env=env).strip()
-
-
-def run_cmd_default(cmd: list[str], default: str = "", cwd=None, env=None) -> str:
-  try:
-    return run_cmd(cmd, cwd=cwd, env=env)
-  except subprocess.CalledProcessError:
-    return default
-
-
-@contextmanager
-def managed_proc(cmd: list[str], env: dict[str, str]):
-  proc = Popen(cmd, env=env, stdout=PIPE, stderr=PIPE)
-  try:
-    yield proc
-  finally:
-    if proc.poll() is None:
-      proc.terminate()
-    try:
-      proc.wait(timeout=5)
-    except TimeoutExpired:
-      proc.kill()

common/tests/test_file_helpers.py

@@ -1,7 +1,7 @@
 import os
 from uuid import uuid4
 
-from openpilot.common.file_helpers import atomic_write_in_dir
+from openpilot.common.utils import atomic_write_in_dir
 
 
 class TestFileHelpers:

common/utils.py

@@ -2,7 +2,12 @@ import io
 import os
 import tempfile
 import contextlib
+import subprocess
+import time
+import functools
+from subprocess import Popen, PIPE, TimeoutExpired
 import zstandard as zstd
+from openpilot.common.swaglog import cloudlog
 
 LOG_COMPRESSION_LEVEL = 10  # little benefit up to level 15. level ~17 is a small step change
 
@@ -27,7 +32,7 @@ class CallbackReader:
 
 
 @contextlib.contextmanager
-def atomic_write_in_dir(path: str, mode: str = 'w', buffering: int = -1, encoding: str = None, newline: str = None,
+def atomic_write_in_dir(path: str, mode: str = 'w', buffering: int = -1, encoding: str | None = None, newline: str | None = None,
                         overwrite: bool = False):
   """Write to a file atomically using a temporary file in the same directory as the destination file."""
   dir_name = os.path.dirname(path)
@@ -56,3 +61,58 @@ def get_upload_stream(filepath: str, should_compress: bool) -> tuple[io.Buffered
     compressed_size = compressed_stream.tell()
     compressed_stream.seek(0)
     return compressed_stream, compressed_size
+
+
+# remove all keys that end in DEPRECATED
+def strip_deprecated_keys(d):
+  for k in list(d.keys()):
+    if isinstance(k, str):
+      if k.endswith('DEPRECATED'):
+        d.pop(k)
+      elif isinstance(d[k], dict):
+        strip_deprecated_keys(d[k])
+  return d
+
+
+def run_cmd(cmd: list[str], cwd=None, env=None) -> str:
+  return subprocess.check_output(cmd, encoding='utf8', cwd=cwd, env=env).strip()
+
+
+def run_cmd_default(cmd: list[str], default: str = "", cwd=None, env=None) -> str:
+  try:
+    return run_cmd(cmd, cwd=cwd, env=env)
+  except subprocess.CalledProcessError:
+    return default
+
+
+@contextlib.contextmanager
+def managed_proc(cmd: list[str], env: dict[str, str]):
+  proc = Popen(cmd, env=env, stdout=PIPE, stderr=PIPE)
+  try:
+    yield proc
+  finally:
+    if proc.poll() is None:
+      proc.terminate()
+    try:
+      proc.wait(timeout=5)
+    except TimeoutExpired:
+      proc.kill()
+
+
+def retry(attempts=3, delay=1.0, ignore_failure=False):
+  def decorator(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+      for _ in range(attempts):
+        try:
+          return func(*args, **kwargs)
+        except Exception:
+          cloudlog.exception(f"{func.__name__} failed, trying again")
+          time.sleep(delay)
+
+      if ignore_failure:
+        cloudlog.error(f"{func.__name__} failed after retry")
+      else:
+        raise Exception(f"{func.__name__} failed after retry")
+    return wrapper
+  return decorator

selfdrive/debug/qlog_size.py

@@ -6,7 +6,7 @@ from collections import defaultdict
 import matplotlib.pyplot as plt
 
 from cereal.services import SERVICE_LIST
-from openpilot.common.file_helpers import LOG_COMPRESSION_LEVEL
+from openpilot.common.utils import LOG_COMPRESSION_LEVEL
 from openpilot.tools.lib.logreader import LogReader
 from tqdm import tqdm
 

selfdrive/pandad/tests/test_pandad_loopback.py

@@ -9,7 +9,7 @@ from pprint import pprint
 import cereal.messaging as messaging
 from cereal import car, log
 from opendbc.car.can_definitions import CanData
-from openpilot.common.retry import retry
+from openpilot.common.utils import retry
 from openpilot.common.params import Params
 from openpilot.common.timeout import Timeout
 from openpilot.selfdrive.pandad import can_list_to_can_capnp

selfdrive/ui/soundd.py

@@ -8,7 +8,7 @@ from cereal import car, messaging
 from openpilot.common.basedir import BASEDIR
 from openpilot.common.filter_simple import FirstOrderFilter
 from openpilot.common.realtime import Ratekeeper
-from openpilot.common.retry import retry
+from openpilot.common.utils import retry
 from openpilot.common.swaglog import cloudlog
 
 from openpilot.system import micd

system/athena/athenad.py

@@ -31,7 +31,7 @@ import cereal.messaging as messaging
 from cereal import log
 from cereal.services import SERVICE_LIST
 from openpilot.common.api import Api
-from openpilot.common.file_helpers import CallbackReader, get_upload_stream
+from openpilot.common.utils import CallbackReader, get_upload_stream
 from openpilot.common.params import Params
 from openpilot.common.realtime import set_core_affinity
 from openpilot.system.hardware import HARDWARE, PC

system/hardware/hardwared.py

@@ -12,7 +12,7 @@ import psutil
 import cereal.messaging as messaging
 from cereal import log
 from cereal.services import SERVICE_LIST
-from openpilot.common.dict_helpers import strip_deprecated_keys
+from openpilot.common.utils import strip_deprecated_keys
 from openpilot.common.filter_simple import FirstOrderFilter
 from openpilot.common.params import Params
 from openpilot.common.realtime import DT_HW

system/loggerd/uploader.py

@@ -12,7 +12,7 @@ from collections.abc import Iterator
 from cereal import log
 import cereal.messaging as messaging
 from openpilot.common.api import Api
-from openpilot.common.file_helpers import get_upload_stream
+from openpilot.common.utils import get_upload_stream
 from openpilot.common.params import Params
 from openpilot.common.realtime import set_core_affinity
 from openpilot.system.hardware.hw import Paths

system/micd.py

@@ -5,7 +5,7 @@ import threading
 
 from cereal import messaging
 from openpilot.common.realtime import Ratekeeper
-from openpilot.common.retry import retry
+from openpilot.common.utils import retry
 from openpilot.common.swaglog import cloudlog
 
 RATE = 10

system/qcomgpsd/qcomgpsd.py

@@ -16,7 +16,7 @@ from struct import unpack_from, calcsize, pack
 from cereal import log
 import cereal.messaging as messaging
 from openpilot.common.gpio import gpio_init, gpio_set
-from openpilot.common.retry import retry
+from openpilot.common.utils import retry
 from openpilot.common.time_helpers import system_time_valid
 from openpilot.system.hardware.tici.pins import GPIO
 from openpilot.common.swaglog import cloudlog

system/statsd.py

@@ -13,7 +13,7 @@ from cereal.messaging import SubMaster
 from openpilot.system.hardware.hw import Paths
 from openpilot.common.swaglog import cloudlog
 from openpilot.system.hardware import HARDWARE
-from openpilot.common.file_helpers import atomic_write_in_dir
+from openpilot.common.utils import atomic_write_in_dir
 from openpilot.system.version import get_build_metadata
 from openpilot.system.loggerd.config import STATS_DIR_FILE_LIMIT, STATS_SOCKET, STATS_FLUSH_TIME_S
 

system/ui/setup.py

@@ -11,7 +11,7 @@ import shutil
 import pyray as rl
 
 from cereal import log
-from openpilot.common.run import run_cmd
+from openpilot.common.utils import run_cmd
 from openpilot.system.hardware import HARDWARE
 from openpilot.system.ui.lib.scroll_panel import GuiScrollPanel
 from openpilot.system.ui.lib.application import gui_app, FontWeight, FONT_SCALE

tools/clip/run.py

@@ -17,7 +17,7 @@ from cereal.messaging import SubMaster
 from openpilot.common.basedir import BASEDIR
 from openpilot.common.params import Params, UnknownKeyName
 from openpilot.common.prefix import OpenpilotPrefix
-from openpilot.common.run import managed_proc
+from openpilot.common.utils import managed_proc
 from openpilot.tools.lib.route import Route
 from openpilot.tools.lib.logreader import LogReader
 

tools/lib/filereader.py

@@ -2,7 +2,7 @@ import os
 import posixpath
 import socket
 from functools import cache
-from openpilot.common.retry import retry
+from openpilot.common.utils import retry
 from urllib.parse import urlparse
 
 from openpilot.tools.lib.url_file import URLFile

tools/lib/url_file.py

@@ -7,7 +7,7 @@ from urllib3 import PoolManager, Retry
 from urllib3.response import BaseHTTPResponse
 from urllib3.util import Timeout
 
-from openpilot.common.file_helpers import atomic_write_in_dir
+from openpilot.common.utils import atomic_write_in_dir
 from openpilot.system.hardware.hw import Paths
 
 #  Cache chunk size