diff --git a/common/params.cc b/common/params.cc index 288de76b0c..fb672cd8aa 100644 --- a/common/params.cc +++ b/common/params.cc @@ -177,6 +177,7 @@ std::unordered_map keys = { {"Offroad_UpdateFailed", CLEAR_ON_MANAGER_START}, {"OpenpilotEnabledToggle", PERSISTENT}, {"PandaHeartbeatLost", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION}, + {"PandaLogState", PERSISTENT}, {"PandaSignatures", CLEAR_ON_MANAGER_START}, {"Passive", PERSISTENT}, {"PrimeType", PERSISTENT}, diff --git a/selfdrive/boardd/pandad.py b/selfdrive/boardd/pandad.py index 4d9b4d8960..5f308b9ad9 100755 --- a/selfdrive/boardd/pandad.py +++ b/selfdrive/boardd/pandad.py @@ -3,6 +3,7 @@ import os import usb1 import time +import json import subprocess from typing import List, NoReturn from functools import cmp_to_key @@ -23,6 +24,48 @@ def get_expected_signature(panda: Panda) -> bytes: cloudlog.exception("Error computing expected signature") return b"" +def read_panda_logs(panda: Panda) -> None: + """ + Forward panda logs to the cloud + """ + + params = Params() + serial = panda.get_usb_serial() + + log_state = {} + try: + l = json.loads(params.get("PandaLogState")) + for k, v in l.items(): + if isinstance(k, str) and isinstance(v, int): + log_state[k] = v + except (TypeError, json.JSONDecodeError): + cloudlog.exception("failed to parse PandaLogState") + + try: + if serial in log_state: + logs = panda.get_logs(last_id=log_state[serial]) + else: + logs = panda.get_logs(get_all=True) + + # truncate logs to 100 entries if needed + MAX_LOGS = 100 + if len(logs) > MAX_LOGS: + cloudlog.warning(f"Panda {serial} has {len(logs)} logs, truncating to {MAX_LOGS}") + logs = logs[-MAX_LOGS:] + + # update log state + if len(logs) > 0: + log_state[serial] = logs[-1]["id"] + + for log in logs: + if log['timestamp'] is not None: + log['timestamp'] = log['timestamp'].isoformat() + cloudlog.event("panda_log", **log, serial=serial) + + params.put("PandaLogState", json.dumps(log_state)) + except Exception: + cloudlog.exception(f"Error getting logs for panda {serial}") + def flash_panda(panda_serial: str) -> Panda: try: @@ -47,7 +90,6 @@ def flash_panda(panda_serial: str) -> Panda: if panda.bootstub: bootstub_version = panda.get_version() cloudlog.info(f"Flashed firmware not booting, flashing development bootloader. {bootstub_version=}, {internal_panda=}") - if internal_panda: HARDWARE.recover_internal_panda() panda.recover(reset=(not internal_panda)) @@ -139,6 +181,8 @@ def main() -> NoReturn: params.put_bool("PandaHeartbeatLost", True) cloudlog.event("heartbeat lost", deviceState=health, serial=panda.get_usb_serial()) + read_panda_logs(panda) + if first_run: if panda.is_internal(): # update time from RTC diff --git a/selfdrive/boardd/tests/test_pandad.py b/selfdrive/boardd/tests/test_pandad.py index c1f080efe5..1d49446bf5 100755 --- a/selfdrive/boardd/tests/test_pandad.py +++ b/selfdrive/boardd/tests/test_pandad.py @@ -6,6 +6,7 @@ import unittest import cereal.messaging as messaging from cereal import log from common.gpio import gpio_set, gpio_init +from common.params import Params from panda import Panda, PandaDFU, PandaProtocolMismatch from selfdrive.test.helpers import phone_only from selfdrive.manager.process_config import managed_processes @@ -17,6 +18,10 @@ HERE = os.path.dirname(os.path.realpath(__file__)) class TestPandad(unittest.TestCase): + def setUp(self): + self.params = Params() + self.start_log_state = self.params.get("PandaLogState") + def tearDown(self): managed_processes['pandad'].stop() @@ -30,6 +35,10 @@ class TestPandad(unittest.TestCase): if sm['peripheralState'].pandaType == log.PandaState.PandaType.unknown: raise Exception("boardd failed to start") + # simple check that we did something with the panda logs + cur_log_state = self.params.get("PandaLogState") + assert cur_log_state != self.start_log_state + def _go_to_dfu(self): HARDWARE.recover_internal_panda() assert Panda.wait_for_dfu(None, 10)