pandad: wait for internal panda to come up (#27705)

* pandad: wait for internal panda to come up

* test

* more checks

selfdrive/boardd/pandad.py

@@ -115,6 +115,14 @@ def main() -> NoReturn:
           cloudlog.info(f"Resetting panda {panda.get_usb_serial()}")
           panda.reset()
 
+      # Ensure internal panda is present if expected
+      internal_pandas = [panda for panda in pandas if panda.is_internal()]
+      if HARDWARE.has_internal_panda() and len(internal_pandas) == 0:
+        cloudlog.error("Internal panda is missing, resetting")
+        HARDWARE.reset_internal_panda()
+        time.sleep(2)  # wait to come back up
+        continue
+
       # sort pandas to have deterministic order
       pandas.sort(key=cmp_to_key(panda_sort_cmp))
       panda_serials = list(map(lambda p: p.get_usb_serial(), pandas))  # type: ignore

selfdrive/boardd/tests/test_pandad.py

@@ -4,9 +4,11 @@ import unittest
 
 import cereal.messaging as messaging
 from panda import Panda
+from common.gpio import gpio_set, gpio_init
 from selfdrive.test.helpers import phone_only
 from selfdrive.manager.process_config import managed_processes
 from system.hardware import HARDWARE
+from system.hardware.tici.pins import GPIO
 
 
 class TestPandad(unittest.TestCase):
@@ -40,6 +42,18 @@ class TestPandad(unittest.TestCase):
     managed_processes['pandad'].start()
     self._wait_for_boardd()
 
+  @phone_only
+  def test_internal_panda_reset(self):
+    gpio_init(GPIO.STM_RST_N, True)
+    gpio_set(GPIO.STM_RST_N, 1)
+    time.sleep(0.5)
+    assert all(not Panda(s).is_internal() for s in Panda.list())
+
+    managed_processes['pandad'].start()
+    self._wait_for_boardd()
+
+    assert any(Panda(s).is_internal() for s in Panda.list())
+
   #def test_out_of_date_fw(self):
   #  pass
 

system/hardware/base.py

@@ -135,6 +135,9 @@ class HardwareBase(ABC):
   def get_networks(self):
     pass
 
+  def has_internal_panda(self) -> bool:
+    return False
+
   def reset_internal_panda(self):
     pass
 

system/hardware/tici/hardware.py

@@ -565,6 +565,9 @@ class Tici(HardwareBase):
     except Exception:
       return -1, -1
 
+  def has_internal_panda(self):
+    return True
+
   def reset_internal_panda(self):
     gpio_init(GPIO.STM_RST_N, True)