pandad: better handling of internal panda failures (#23755)

* reset internal panda

* recover

* internal dfu

Co-authored-by: Comma Device <device@comma.ai>

common/gpio.py

@@ -1,4 +1,4 @@
-def gpio_init(pin, output):
+def gpio_init(pin: int, output: bool) -> None:
   try:
     with open(f"/sys/class/gpio/gpio{pin}/direction", 'wb') as f:
       f.write(b"out" if output else b"in")
@@ -6,7 +6,7 @@ def gpio_init(pin, output):
     print(f"Failed to set gpio {pin} direction: {e}")
 
 
-def gpio_set(pin, high):
+def gpio_set(pin: int, high: bool) -> None:
   try:
     with open(f"/sys/class/gpio/gpio{pin}/value", 'wb') as f:
       f.write(b"1" if high else b"0")

panda

@@ -1 +1 @@
-Subproject commit 7dd9493eb102ba8b96362c7265b06851ec1d4bac
+Subproject commit 5eefc3ad62422dea3fc356f82267af58f9b1518a

selfdrive/boardd/pandad.py

@@ -10,6 +10,7 @@ from functools import cmp_to_key
 from panda import DEFAULT_FW_FN, DEFAULT_H7_FW_FN, MCU_TYPE_H7, Panda, PandaDFU
 from common.basedir import BASEDIR
 from common.params import Params
+from selfdrive.hardware import HARDWARE
 from selfdrive.swaglog import cloudlog
 
 
@@ -27,6 +28,7 @@ def flash_panda(panda_serial: str) -> Panda:
   panda = Panda(panda_serial)
 
   fw_signature = get_expected_signature(panda)
+  internal_panda = panda.is_internal() and not panda.bootstub
 
   panda_version = "bootstub" if panda.bootstub else panda.get_version()
   panda_signature = b"" if panda.bootstub else panda.get_signature()
@@ -40,7 +42,9 @@ def flash_panda(panda_serial: str) -> Panda:
   if panda.bootstub:
     bootstub_version = panda.get_version()
     cloudlog.info(f"Flashed firmware not booting, flashing development bootloader. Bootstub version: {bootstub_version}")
-    panda.recover()
+    if internal_panda:
+      HARDWARE.recover_internal_panda()
+    panda.recover(reset=(not internal_panda))
     cloudlog.info("Done flashing bootloader")
 
   if panda.bootstub:
@@ -89,6 +93,10 @@ def main() -> NoReturn:
 
       panda_serials = Panda.list()
       if len(panda_serials) == 0:
+        if first_run:
+          cloudlog.info("Resetting internal panda")
+          HARDWARE.reset_internal_panda()
+          time.sleep(2)  # wait to come back up
         continue
 
       cloudlog.info(f"{len(panda_serials)} panda(s) found, connecting - {panda_serials}")

selfdrive/hardware/base.py

@@ -158,3 +158,9 @@ class HardwareBase(ABC):
   @abstractmethod
   def get_networks(self):
     pass
+
+  def reset_internal_panda(self):
+    pass
+
+  def recover_internal_panda(self):
+    pass

selfdrive/hardware/tici/hardware.py

@@ -2,13 +2,16 @@ import json
 import math
 import os
 import subprocess
+import time
 from enum import IntEnum
 from functools import cached_property
 from pathlib import Path
 
 from cereal import log
+from common.gpio import gpio_set, gpio_init
 from selfdrive.hardware.base import HardwareBase, ThermalConfig
 from selfdrive.hardware.tici import iwlist
+from selfdrive.hardware.tici.pins import GPIO
 from selfdrive.hardware.tici.amplifier import Amplifier
 
 NM = 'org.freedesktop.NetworkManager'
@@ -511,6 +514,24 @@ class Tici(HardwareBase):
 
     return r
 
+  def reset_internal_panda(self):
+    gpio_init(GPIO.STM_RST_N, True)
+
+    gpio_set(GPIO.STM_RST_N, 1)
+    time.sleep(2)
+    gpio_set(GPIO.STM_RST_N, 0)
+
+
+  def recover_internal_panda(self):
+    gpio_init(GPIO.STM_RST_N, True)
+    gpio_init(GPIO.STM_BOOT0, True)
+
+    gpio_set(GPIO.STM_RST_N, 1)
+    gpio_set(GPIO.STM_BOOT0, 1)
+    time.sleep(2)
+    gpio_set(GPIO.STM_RST_N, 0)
+    gpio_set(GPIO.STM_BOOT0, 0)
+
 
 if __name__ == "__main__":
   t = Tici()

selfdrive/hardware/tici/pins.py

@@ -1,4 +1,5 @@
 # TODO: these are also defined in a header
+
 # GPIO pin definitions
 class GPIO:
   # both GPIO_STM_RST_N and GPIO_LTE_RST_N are misnamed, they are high to reset
@@ -18,5 +19,3 @@ class GPIO:
   CAM0_RSTN = 9
   CAM1_RSTN = 7
   CAM2_RSTN = 12
-
-