|
|
@ -17,7 +17,6 @@ from common.basedir import BASEDIR |
|
|
|
from common.params import Params |
|
|
|
from common.params import Params |
|
|
|
from common.realtime import sec_since_boot |
|
|
|
from common.realtime import sec_since_boot |
|
|
|
from system.swaglog import cloudlog |
|
|
|
from system.swaglog import cloudlog |
|
|
|
from system.hardware import HARDWARE |
|
|
|
|
|
|
|
from cereal import log |
|
|
|
from cereal import log |
|
|
|
|
|
|
|
|
|
|
|
WATCHDOG_FN = "/dev/shm/wd_" |
|
|
|
WATCHDOG_FN = "/dev/shm/wd_" |
|
|
@ -67,7 +66,6 @@ def join_process(process: Process, timeout: float) -> None: |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class ManagerProcess(ABC): |
|
|
|
class ManagerProcess(ABC): |
|
|
|
unkillable = False |
|
|
|
|
|
|
|
daemon = False |
|
|
|
daemon = False |
|
|
|
sigkill = False |
|
|
|
sigkill = False |
|
|
|
onroad = True |
|
|
|
onroad = True |
|
|
@ -132,22 +130,11 @@ class ManagerProcess(ABC): |
|
|
|
|
|
|
|
|
|
|
|
join_process(self.proc, 5) |
|
|
|
join_process(self.proc, 5) |
|
|
|
|
|
|
|
|
|
|
|
# If process failed to die send SIGKILL or reboot |
|
|
|
# If process failed to die send SIGKILL |
|
|
|
if self.proc.exitcode is None and retry: |
|
|
|
if self.proc.exitcode is None and retry: |
|
|
|
if self.unkillable: |
|
|
|
cloudlog.info(f"killing {self.name} with SIGKILL") |
|
|
|
cloudlog.critical(f"unkillable process {self.name} failed to exit! rebooting in 15 if it doesn't die") |
|
|
|
self.signal(signal.SIGKILL) |
|
|
|
join_process(self.proc, 15) |
|
|
|
self.proc.join() |
|
|
|
|
|
|
|
|
|
|
|
if self.proc.exitcode is None: |
|
|
|
|
|
|
|
cloudlog.critical(f"unkillable process {self.name} failed to die!") |
|
|
|
|
|
|
|
os.system("date >> /data/unkillable_reboot") |
|
|
|
|
|
|
|
os.sync() |
|
|
|
|
|
|
|
HARDWARE.reboot() |
|
|
|
|
|
|
|
raise RuntimeError |
|
|
|
|
|
|
|
else: |
|
|
|
|
|
|
|
cloudlog.info(f"killing {self.name} with SIGKILL") |
|
|
|
|
|
|
|
self.signal(signal.SIGKILL) |
|
|
|
|
|
|
|
self.proc.join() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ret = self.proc.exitcode |
|
|
|
ret = self.proc.exitcode |
|
|
|
cloudlog.info(f"{self.name} is dead with {ret}") |
|
|
|
cloudlog.info(f"{self.name} is dead with {ret}") |
|
|
|