|
|
|
@ -1,7 +1,6 @@ |
|
|
|
import importlib |
|
|
|
import importlib |
|
|
|
import os |
|
|
|
import os |
|
|
|
import signal |
|
|
|
import signal |
|
|
|
import struct |
|
|
|
|
|
|
|
import time |
|
|
|
import time |
|
|
|
import subprocess |
|
|
|
import subprocess |
|
|
|
from collections.abc import Callable, ValuesView |
|
|
|
from collections.abc import Callable, ValuesView |
|
|
|
@ -16,9 +15,6 @@ import openpilot.system.sentry as sentry |
|
|
|
from openpilot.common.basedir import BASEDIR |
|
|
|
from openpilot.common.basedir import BASEDIR |
|
|
|
from openpilot.common.params import Params |
|
|
|
from openpilot.common.params import Params |
|
|
|
from openpilot.common.swaglog import cloudlog |
|
|
|
from openpilot.common.swaglog import cloudlog |
|
|
|
from openpilot.common.watchdog import WATCHDOG_FN |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ENABLE_WATCHDOG = os.getenv("NO_WATCHDOG") is None |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def launcher(proc: str, name: str) -> None: |
|
|
|
def launcher(proc: str, name: str) -> None: |
|
|
|
@ -70,10 +66,6 @@ class ManagerProcess(ABC): |
|
|
|
proc: Process | None = None |
|
|
|
proc: Process | None = None |
|
|
|
enabled = True |
|
|
|
enabled = True |
|
|
|
name = "" |
|
|
|
name = "" |
|
|
|
|
|
|
|
|
|
|
|
last_watchdog_time = 0 |
|
|
|
|
|
|
|
watchdog_max_dt: int | None = None |
|
|
|
|
|
|
|
watchdog_seen = False |
|
|
|
|
|
|
|
shutting_down = False |
|
|
|
shutting_down = False |
|
|
|
|
|
|
|
|
|
|
|
@abstractmethod |
|
|
|
@abstractmethod |
|
|
|
@ -88,26 +80,6 @@ class ManagerProcess(ABC): |
|
|
|
self.stop(sig=signal.SIGKILL) |
|
|
|
self.stop(sig=signal.SIGKILL) |
|
|
|
self.start() |
|
|
|
self.start() |
|
|
|
|
|
|
|
|
|
|
|
def check_watchdog(self, started: bool) -> None: |
|
|
|
|
|
|
|
if self.watchdog_max_dt is None or self.proc is None: |
|
|
|
|
|
|
|
return |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
try: |
|
|
|
|
|
|
|
fn = WATCHDOG_FN + str(self.proc.pid) |
|
|
|
|
|
|
|
with open(fn, "rb") as f: |
|
|
|
|
|
|
|
self.last_watchdog_time = struct.unpack('Q', f.read())[0] |
|
|
|
|
|
|
|
except Exception: |
|
|
|
|
|
|
|
pass |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
dt = time.monotonic() - self.last_watchdog_time / 1e9 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if dt > self.watchdog_max_dt: |
|
|
|
|
|
|
|
if self.watchdog_seen and ENABLE_WATCHDOG: |
|
|
|
|
|
|
|
cloudlog.error(f"Watchdog timeout for {self.name} (exitcode {self.proc.exitcode}) restarting ({started=})") |
|
|
|
|
|
|
|
self.restart() |
|
|
|
|
|
|
|
else: |
|
|
|
|
|
|
|
self.watchdog_seen = True |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def stop(self, retry: bool = True, block: bool = True, sig: signal.Signals = None) -> int | None: |
|
|
|
def stop(self, retry: bool = True, block: bool = True, sig: signal.Signals = None) -> int | None: |
|
|
|
if self.proc is None: |
|
|
|
if self.proc is None: |
|
|
|
return None |
|
|
|
return None |
|
|
|
@ -167,14 +139,13 @@ class ManagerProcess(ABC): |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class NativeProcess(ManagerProcess): |
|
|
|
class NativeProcess(ManagerProcess): |
|
|
|
def __init__(self, name, cwd, cmdline, should_run, enabled=True, sigkill=False, watchdog_max_dt=None): |
|
|
|
def __init__(self, name, cwd, cmdline, should_run, enabled=True, sigkill=False): |
|
|
|
self.name = name |
|
|
|
self.name = name |
|
|
|
self.cwd = cwd |
|
|
|
self.cwd = cwd |
|
|
|
self.cmdline = cmdline |
|
|
|
self.cmdline = cmdline |
|
|
|
self.should_run = should_run |
|
|
|
self.should_run = should_run |
|
|
|
self.enabled = enabled |
|
|
|
self.enabled = enabled |
|
|
|
self.sigkill = sigkill |
|
|
|
self.sigkill = sigkill |
|
|
|
self.watchdog_max_dt = watchdog_max_dt |
|
|
|
|
|
|
|
self.launcher = nativelauncher |
|
|
|
self.launcher = nativelauncher |
|
|
|
|
|
|
|
|
|
|
|
def prepare(self) -> None: |
|
|
|
def prepare(self) -> None: |
|
|
|
@ -192,18 +163,16 @@ class NativeProcess(ManagerProcess): |
|
|
|
cloudlog.info(f"starting process {self.name}") |
|
|
|
cloudlog.info(f"starting process {self.name}") |
|
|
|
self.proc = Process(name=self.name, target=self.launcher, args=(self.cmdline, cwd, self.name)) |
|
|
|
self.proc = Process(name=self.name, target=self.launcher, args=(self.cmdline, cwd, self.name)) |
|
|
|
self.proc.start() |
|
|
|
self.proc.start() |
|
|
|
self.watchdog_seen = False |
|
|
|
|
|
|
|
self.shutting_down = False |
|
|
|
self.shutting_down = False |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class PythonProcess(ManagerProcess): |
|
|
|
class PythonProcess(ManagerProcess): |
|
|
|
def __init__(self, name, module, should_run, enabled=True, sigkill=False, watchdog_max_dt=None): |
|
|
|
def __init__(self, name, module, should_run, enabled=True, sigkill=False): |
|
|
|
self.name = name |
|
|
|
self.name = name |
|
|
|
self.module = module |
|
|
|
self.module = module |
|
|
|
self.should_run = should_run |
|
|
|
self.should_run = should_run |
|
|
|
self.enabled = enabled |
|
|
|
self.enabled = enabled |
|
|
|
self.sigkill = sigkill |
|
|
|
self.sigkill = sigkill |
|
|
|
self.watchdog_max_dt = watchdog_max_dt |
|
|
|
|
|
|
|
self.launcher = launcher |
|
|
|
self.launcher = launcher |
|
|
|
|
|
|
|
|
|
|
|
def prepare(self) -> None: |
|
|
|
def prepare(self) -> None: |
|
|
|
@ -226,7 +195,6 @@ class PythonProcess(ManagerProcess): |
|
|
|
cloudlog.info(f"starting python {self.module}") |
|
|
|
cloudlog.info(f"starting python {self.module}") |
|
|
|
self.proc = Process(name=name, target=self.launcher, args=(self.module, self.name)) |
|
|
|
self.proc = Process(name=name, target=self.launcher, args=(self.module, self.name)) |
|
|
|
self.proc.start() |
|
|
|
self.proc.start() |
|
|
|
self.watchdog_seen = False |
|
|
|
|
|
|
|
self.shutting_down = False |
|
|
|
self.shutting_down = False |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -288,8 +256,6 @@ def ensure_running(procs: ValuesView[ManagerProcess], started: bool, params=None |
|
|
|
else: |
|
|
|
else: |
|
|
|
p.stop(block=False) |
|
|
|
p.stop(block=False) |
|
|
|
|
|
|
|
|
|
|
|
p.check_watchdog(started) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
for p in running: |
|
|
|
for p in running: |
|
|
|
p.start() |
|
|
|
p.start() |
|
|
|
|
|
|
|
|
|
|
|
|