pandad: recover from bad bootstub (#29638)

* pandad: recover from bad bootstub

* more

* adjust that

* ugh guess multipanda is still a thing

* reset

---------

Co-authored-by: Comma Device <device@comma.ai>

selfdrive/boardd/pandad.py

@@ -129,6 +129,7 @@ def main() -> NoReturn:
   count = 0
   first_run = True
   params = Params()
+  no_internal_panda_count = 0
 
   while True:
     try:
@@ -136,6 +137,16 @@ def main() -> NoReturn:
       cloudlog.event("pandad.flash_and_connect", count=count)
       params.remove("PandaSignatures")
 
+      # Handle missing internal panda
+      if no_internal_panda_count > 0:
+        if no_internal_panda_count == 3:
+          cloudlog.info("No pandas found, putting internal panda into DFU")
+          HARDWARE.recover_internal_panda()
+        else:
+          cloudlog.info("No pandas found, resetting internal panda")
+          HARDWARE.reset_internal_panda()
+        time.sleep(3)  # wait to come back up
+
       # Flash all Pandas in DFU mode
       dfu_serials = PandaDFU.list()
       if len(dfu_serials) > 0:
@@ -146,10 +157,7 @@ def main() -> NoReturn:
 
       panda_serials = Panda.list()
       if len(panda_serials) == 0:
-        if first_run:
-          cloudlog.info("No pandas found, resetting internal panda")
-          HARDWARE.reset_internal_panda()
-          time.sleep(2)  # wait to come back up
+        no_internal_panda_count += 1
         continue
 
       cloudlog.info(f"{len(panda_serials)} panda(s) found, connecting - {panda_serials}")
@@ -162,10 +170,10 @@ def main() -> NoReturn:
       # Ensure internal panda is present if expected
       internal_pandas = [panda for panda in pandas if panda.is_internal()]
       if HARDWARE.has_internal_panda() and len(internal_pandas) == 0:
-        cloudlog.error("Internal panda is missing, resetting")
-        HARDWARE.reset_internal_panda()
-        time.sleep(2)  # wait to come back up
+        cloudlog.error("Internal panda is missing, trying again")
+        no_internal_panda_count += 1
         continue
+      no_internal_panda_count = 0
 
       # sort pandas to have deterministic order
       pandas.sort(key=cmp_to_key(panda_sort_cmp))

selfdrive/boardd/tests/test_pandad.py

@@ -25,13 +25,17 @@ class TestPandad(unittest.TestCase):
   def tearDown(self):
     managed_processes['pandad'].stop()
 
-  def _wait_for_boardd(self, timeout=30):
+  def _run_test(self, timeout=30):
+    managed_processes['pandad'].start()
+
     sm = messaging.SubMaster(['peripheralState'])
     for _ in range(timeout*10):
       sm.update(100)
       if sm['peripheralState'].pandaType != log.PandaState.PandaType.unknown:
         break
 
+    managed_processes['pandad'].stop()
+
     if sm['peripheralState'].pandaType == log.PandaState.PandaType.unknown:
       raise Exception("boardd failed to start")
 
@@ -43,7 +47,11 @@ class TestPandad(unittest.TestCase):
     HARDWARE.recover_internal_panda()
     assert Panda.wait_for_dfu(None, 10)
 
-  def _flash_and_test(self, fn, expect_mismatch=False):
+  def _assert_no_panda(self):
+    assert not Panda.wait_for_dfu(None, 3)
+    assert not Panda.wait_for_panda(None, 3)
+
+  def _flash_bootstub_and_test(self, fn, expect_mismatch=False):
     self._go_to_dfu()
     pd = PandaDFU(None)
     if fn is None:
@@ -61,22 +69,19 @@ class TestPandad(unittest.TestCase):
       with Panda() as p:
         assert p.bootstub
 
-    managed_processes['pandad'].start()
-    self._wait_for_boardd(45)
+    self._run_test(45)
 
   @phone_only
   def test_in_dfu(self):
     HARDWARE.recover_internal_panda()
-    managed_processes['pandad'].start()
-    self._wait_for_boardd(60)
+    self._run_test(60)
 
   @phone_only
   def test_in_bootstub(self):
     with Panda() as p:
       p.reset(enter_bootstub=True)
       assert p.bootstub
-    managed_processes['pandad'].start()
-    self._wait_for_boardd()
+    self._run_test()
 
   @phone_only
   def test_internal_panda_reset(self):
@@ -84,22 +89,17 @@ class TestPandad(unittest.TestCase):
     gpio_set(GPIO.STM_RST_N, 1)
     time.sleep(0.5)
     assert all(not Panda(s).is_internal() for s in Panda.list())
-
-    managed_processes['pandad'].start()
-    self._wait_for_boardd()
+    self._run_test()
 
     assert any(Panda(s).is_internal() for s in Panda.list())
 
   @phone_only
   def test_best_case_startup_time(self):
     # run once so we're setup
-    managed_processes['pandad'].start()
-    self._wait_for_boardd()
-    managed_processes['pandad'].stop()
+    self._run_test()
 
     # should be fast this time
-    managed_processes['pandad'].start()
-    self._wait_for_boardd(8)
+    self._run_test(8)
 
   @phone_only
   def test_protocol_version_check(self):
@@ -108,11 +108,23 @@ class TestPandad(unittest.TestCase):
 
     # flash old fw
     fn = os.path.join(HERE, "bootstub.panda_h7_spiv0.bin")
-    self._flash_and_test(fn, expect_mismatch=True)
+    self._flash_bootstub_and_test(fn, expect_mismatch=True)
 
   @phone_only
   def test_release_to_devel_bootstub(self):
-    self._flash_and_test(None)
+    self._flash_bootstub_and_test(None)
+
+  @phone_only
+  def test_recover_from_bad_bootstub(self):
+    self._go_to_dfu()
+    with PandaDFU(None) as pd:
+      pd.program_bootstub(b"\x00"*1024)
+      pd.reset()
+    HARDWARE.reset_internal_panda()
+    self._assert_no_panda()
+
+    self._run_test(60)
+
 
 if __name__ == "__main__":
   unittest.main()

system/hardware/tici/hardware.py

@@ -586,7 +586,7 @@ class Tici(HardwareBase):
     gpio_init(GPIO.STM_RST_N, True)
 
     gpio_set(GPIO.STM_RST_N, 1)
-    time.sleep(2)
+    time.sleep(1)
     gpio_set(GPIO.STM_RST_N, 0)
 
   def recover_internal_panda(self):
@@ -595,9 +595,9 @@ class Tici(HardwareBase):
 
     gpio_set(GPIO.STM_RST_N, 1)
     gpio_set(GPIO.STM_BOOT0, 1)
-    time.sleep(1)
+    time.sleep(0.5)
     gpio_set(GPIO.STM_RST_N, 0)
-    time.sleep(1)
+    time.sleep(0.5)
     gpio_set(GPIO.STM_BOOT0, 0)