diff --git a/selfdrive/boardd/pandad.py b/selfdrive/boardd/pandad.py index ce9137a30b..e614432fad 100755 --- a/selfdrive/boardd/pandad.py +++ b/selfdrive/boardd/pandad.py @@ -129,6 +129,7 @@ def main() -> NoReturn: count = 0 first_run = True params = Params() + no_internal_panda_count = 0 while True: try: @@ -136,6 +137,16 @@ def main() -> NoReturn: cloudlog.event("pandad.flash_and_connect", count=count) params.remove("PandaSignatures") + # Handle missing internal panda + if no_internal_panda_count > 0: + if no_internal_panda_count == 3: + cloudlog.info("No pandas found, putting internal panda into DFU") + HARDWARE.recover_internal_panda() + else: + cloudlog.info("No pandas found, resetting internal panda") + HARDWARE.reset_internal_panda() + time.sleep(3) # wait to come back up + # Flash all Pandas in DFU mode dfu_serials = PandaDFU.list() if len(dfu_serials) > 0: @@ -146,10 +157,7 @@ def main() -> NoReturn: panda_serials = Panda.list() if len(panda_serials) == 0: - if first_run: - cloudlog.info("No pandas found, resetting internal panda") - HARDWARE.reset_internal_panda() - time.sleep(2) # wait to come back up + no_internal_panda_count += 1 continue cloudlog.info(f"{len(panda_serials)} panda(s) found, connecting - {panda_serials}") @@ -162,10 +170,10 @@ def main() -> NoReturn: # Ensure internal panda is present if expected internal_pandas = [panda for panda in pandas if panda.is_internal()] if HARDWARE.has_internal_panda() and len(internal_pandas) == 0: - cloudlog.error("Internal panda is missing, resetting") - HARDWARE.reset_internal_panda() - time.sleep(2) # wait to come back up + cloudlog.error("Internal panda is missing, trying again") + no_internal_panda_count += 1 continue + no_internal_panda_count = 0 # sort pandas to have deterministic order pandas.sort(key=cmp_to_key(panda_sort_cmp)) diff --git a/selfdrive/boardd/tests/test_pandad.py b/selfdrive/boardd/tests/test_pandad.py index 970af385d8..30a4a9b868 100755 --- a/selfdrive/boardd/tests/test_pandad.py +++ b/selfdrive/boardd/tests/test_pandad.py @@ -25,13 +25,17 @@ class TestPandad(unittest.TestCase): def tearDown(self): managed_processes['pandad'].stop() - def _wait_for_boardd(self, timeout=30): + def _run_test(self, timeout=30): + managed_processes['pandad'].start() + sm = messaging.SubMaster(['peripheralState']) for _ in range(timeout*10): sm.update(100) if sm['peripheralState'].pandaType != log.PandaState.PandaType.unknown: break + managed_processes['pandad'].stop() + if sm['peripheralState'].pandaType == log.PandaState.PandaType.unknown: raise Exception("boardd failed to start") @@ -43,7 +47,11 @@ class TestPandad(unittest.TestCase): HARDWARE.recover_internal_panda() assert Panda.wait_for_dfu(None, 10) - def _flash_and_test(self, fn, expect_mismatch=False): + def _assert_no_panda(self): + assert not Panda.wait_for_dfu(None, 3) + assert not Panda.wait_for_panda(None, 3) + + def _flash_bootstub_and_test(self, fn, expect_mismatch=False): self._go_to_dfu() pd = PandaDFU(None) if fn is None: @@ -61,22 +69,19 @@ class TestPandad(unittest.TestCase): with Panda() as p: assert p.bootstub - managed_processes['pandad'].start() - self._wait_for_boardd(45) + self._run_test(45) @phone_only def test_in_dfu(self): HARDWARE.recover_internal_panda() - managed_processes['pandad'].start() - self._wait_for_boardd(60) + self._run_test(60) @phone_only def test_in_bootstub(self): with Panda() as p: p.reset(enter_bootstub=True) assert p.bootstub - managed_processes['pandad'].start() - self._wait_for_boardd() + self._run_test() @phone_only def test_internal_panda_reset(self): @@ -84,22 +89,17 @@ class TestPandad(unittest.TestCase): gpio_set(GPIO.STM_RST_N, 1) time.sleep(0.5) assert all(not Panda(s).is_internal() for s in Panda.list()) - - managed_processes['pandad'].start() - self._wait_for_boardd() + self._run_test() assert any(Panda(s).is_internal() for s in Panda.list()) @phone_only def test_best_case_startup_time(self): # run once so we're setup - managed_processes['pandad'].start() - self._wait_for_boardd() - managed_processes['pandad'].stop() + self._run_test() # should be fast this time - managed_processes['pandad'].start() - self._wait_for_boardd(8) + self._run_test(8) @phone_only def test_protocol_version_check(self): @@ -108,11 +108,23 @@ class TestPandad(unittest.TestCase): # flash old fw fn = os.path.join(HERE, "bootstub.panda_h7_spiv0.bin") - self._flash_and_test(fn, expect_mismatch=True) + self._flash_bootstub_and_test(fn, expect_mismatch=True) @phone_only def test_release_to_devel_bootstub(self): - self._flash_and_test(None) + self._flash_bootstub_and_test(None) + + @phone_only + def test_recover_from_bad_bootstub(self): + self._go_to_dfu() + with PandaDFU(None) as pd: + pd.program_bootstub(b"\x00"*1024) + pd.reset() + HARDWARE.reset_internal_panda() + self._assert_no_panda() + + self._run_test(60) + if __name__ == "__main__": unittest.main() diff --git a/system/hardware/tici/hardware.py b/system/hardware/tici/hardware.py index 952693dd3a..7f8ef37f1a 100644 --- a/system/hardware/tici/hardware.py +++ b/system/hardware/tici/hardware.py @@ -586,7 +586,7 @@ class Tici(HardwareBase): gpio_init(GPIO.STM_RST_N, True) gpio_set(GPIO.STM_RST_N, 1) - time.sleep(2) + time.sleep(1) gpio_set(GPIO.STM_RST_N, 0) def recover_internal_panda(self): @@ -595,9 +595,9 @@ class Tici(HardwareBase): gpio_set(GPIO.STM_RST_N, 1) gpio_set(GPIO.STM_BOOT0, 1) - time.sleep(1) + time.sleep(0.5) gpio_set(GPIO.STM_RST_N, 0) - time.sleep(1) + time.sleep(0.5) gpio_set(GPIO.STM_BOOT0, 0)