boardd: SPI corruption test (#32404)

* simple test

* little more

---------

Co-authored-by: Comma Device <device@comma.ai>

Jenkinsfile

@@ -237,6 +237,7 @@ node {
         deviceStage("tizi", "tizi", ["UNSAFE=1"], [
           ["build openpilot", "cd selfdrive/manager && ./build.py"],
           ["test boardd loopback", "SINGLE_PANDA=1 pytest selfdrive/boardd/tests/test_boardd_loopback.py"],
+          ["test boardd spi", "pytest selfdrive/boardd/tests/test_boardd_spi.py"],
           ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"],
           ["test amp", "pytest system/hardware/tici/tests/test_amplifier.py"],
           ["test hw", "pytest system/hardware/tici/tests/test_hardware.py"],

selfdrive/boardd/spi.cc

@@ -301,7 +301,11 @@ int PandaSpiHandle::lltransfer(spi_ioc_transfer &t) {
     }
     if ((static_cast<double>(rand()) / RAND_MAX) < err_prob && t.tx_buf != (uint64_t)NULL) {
       printf("corrupting TX\n");
-      memset((uint8_t*)t.tx_buf, (uint8_t)(rand() % 256), rand() % (t.len+1));
+      for (int i = 0; i < t.len; i++) {
+        if ((static_cast<double>(rand()) / RAND_MAX) > 0.9) {
+          ((uint8_t*)t.tx_buf)[i] = (uint8_t)(rand() % 256);
+        }
+      }
     }
   }
 
@@ -310,7 +314,11 @@ int PandaSpiHandle::lltransfer(spi_ioc_transfer &t) {
   if (err_prob > 0) {
     if ((static_cast<double>(rand()) / RAND_MAX) < err_prob && t.rx_buf != (uint64_t)NULL) {
       printf("corrupting RX\n");
-      memset((uint8_t*)t.rx_buf, (uint8_t)(rand() % 256), rand() % (t.len+1));
+      for (int i = 0; i < t.len; i++) {
+        if ((static_cast<double>(rand()) / RAND_MAX) > 0.9) {
+          ((uint8_t*)t.rx_buf)[i] = (uint8_t)(rand() % 256);
+        }
+      }
     }
   }
 

selfdrive/boardd/tests/test_boardd_loopback.py

@@ -3,6 +3,7 @@ import os
 import copy
 import random
 import time
+import pytest
 import unittest
 from collections import defaultdict
 from pprint import pprint
@@ -17,42 +18,61 @@ from openpilot.system.hardware import TICI
 from openpilot.selfdrive.test.helpers import phone_only, with_processes
 
 
-class TestBoardd(unittest.TestCase):
+def setup_boardd(num_pandas):
+  params = Params()
+  params.put_bool("IsOnroad", False)
 
+  with Timeout(90, "boardd didn't start"):
+    sm = messaging.SubMaster(['pandaStates'])
+    while sm.recv_frame['pandaStates'] < 1 or len(sm['pandaStates']) == 0 or \
+        any(ps.pandaType == log.PandaState.PandaType.unknown for ps in sm['pandaStates']):
+      sm.update(1000)
+
+  found_pandas = len(sm['pandaStates'])
+  assert num_pandas == found_pandas, "connected pandas ({found_pandas}) doesn't match expected panda count ({num_pandas}). \
+                                      connect another panda for multipanda tests."
+
+  # boardd safety setting relies on these params
+  cp = car.CarParams.new_message()
+
+  safety_config = car.CarParams.SafetyConfig.new_message()
+  safety_config.safetyModel = car.CarParams.SafetyModel.allOutput
+  cp.safetyConfigs = [safety_config]*num_pandas
+
+  params.put_bool("IsOnroad", True)
+  params.put_bool("FirmwareQueryDone", True)
+  params.put_bool("ControlsReady", True)
+  params.put("CarParams", cp.to_bytes())
+
+
+def send_random_can_messages(sendcan, count, num_pandas=1):
+  sent_msgs = defaultdict(set)
+  for _ in range(count):
+    to_send = []
+    for __ in range(random.randrange(20)):
+      bus = random.choice([b for b in range(3*num_pandas) if b % 4 != 3])
+      addr = random.randrange(1, 1<<29)
+      dat = bytes(random.getrandbits(8) for _ in range(random.randrange(1, 9)))
+      if (addr, dat) in sent_msgs[bus]:
+        continue
+      sent_msgs[bus].add((addr, dat))
+      to_send.append(make_can_msg(addr, dat, bus))
+    sendcan.send(can_list_to_can_capnp(to_send, msgtype='sendcan'))
+  return sent_msgs
+
+
+@pytest.mark.tici
+class TestBoarddLoopback:
   @classmethod
-  def setUpClass(cls):
+  def setup_class(cls):
     os.environ['STARTED'] = '1'
     os.environ['BOARDD_LOOPBACK'] = '1'
 
   @phone_only
   @with_processes(['pandad'])
   def test_loopback(self):
-    params = Params()
+    num_pandas = 2 if TICI and "SINGLE_PANDA" not in os.environ else 1
-    params.put_bool("IsOnroad", False)
+    setup_boardd(num_pandas)
-
-    with Timeout(90, "boardd didn't start"):
-      sm = messaging.SubMaster(['pandaStates'])
-      while sm.recv_frame['pandaStates'] < 1 or len(sm['pandaStates']) == 0 or \
-          any(ps.pandaType == log.PandaState.PandaType.unknown for ps in sm['pandaStates']):
-        sm.update(1000)
-
-    num_pandas = len(sm['pandaStates'])
-    expected_pandas = 2 if TICI and "SINGLE_PANDA" not in os.environ else 1
-    self.assertEqual(num_pandas, expected_pandas, "connected pandas ({num_pandas}) doesn't match expected panda count ({expected_pandas}). \
-                                                   connect another panda for multipanda tests.")
-
-    # boardd safety setting relies on these params
-    cp = car.CarParams.new_message()
-
-    safety_config = car.CarParams.SafetyConfig.new_message()
-    safety_config.safetyModel = car.CarParams.SafetyModel.allOutput
-    cp.safetyConfigs = [safety_config]*num_pandas
-
-    params.put_bool("IsOnroad", True)
-    params.put_bool("FirmwareQueryDone", True)
-    params.put_bool("ControlsReady", True)
-    params.put("CarParams", cp.to_bytes())
-
     sendcan = messaging.pub_sock('sendcan')
     can = messaging.sub_sock('can', conflate=False, timeout=100)
     sm = messaging.SubMaster(['pandaStates'])
@@ -62,16 +82,7 @@ class TestBoardd(unittest.TestCase):
     for i in range(n):
       print(f"boardd loopback {i}/{n}")
 
-      sent_msgs = defaultdict(set)
+      sent_msgs = send_random_can_messages(sendcan, random.randrange(20, 100), num_pandas)
-      for _ in range(random.randrange(20, 100)):
-        to_send = []
-        for __ in range(random.randrange(20)):
-          bus = random.choice([b for b in range(3*num_pandas) if b % 4 != 3])
-          addr = random.randrange(1, 1<<29)
-          dat = bytes(random.getrandbits(8) for _ in range(random.randrange(1, 9)))
-          sent_msgs[bus].add((addr, dat))
-          to_send.append(make_can_msg(addr, dat, bus))
-        sendcan.send(can_list_to_can_capnp(to_send, msgtype='sendcan'))
 
       sent_loopback = copy.deepcopy(sent_msgs)
       sent_loopback.update({k+128: copy.deepcopy(v) for k, v in sent_msgs.items()})

selfdrive/boardd/tests/test_boardd_spi.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+import os
+import time
+import numpy as np
+import pytest
+
+import cereal.messaging as messaging
+from cereal.services import SERVICE_LIST
+from openpilot.system.hardware import HARDWARE
+from openpilot.selfdrive.test.helpers import phone_only, with_processes
+from openpilot.selfdrive.boardd.tests.test_boardd_loopback import setup_boardd
+
+
+@pytest.mark.tici
+class TestBoarddSpi:
+  @classmethod
+  def setup_class(cls):
+    if HARDWARE.get_device_type() == 'tici':
+      pytest.skip("only for spi pandas")
+    os.environ['STARTED'] = '1'
+    os.environ['BOARDD_LOOPBACK'] = '1'
+    os.environ['SPI_ERR_PROB'] = '0.001'
+
+  @phone_only
+  @with_processes(['pandad'])
+  def test_spi_corruption(self, subtests):
+    setup_boardd(1)
+
+    socks = {s: messaging.sub_sock(s, conflate=False, timeout=100) for s in ('can', 'pandaStates', 'peripheralState')}
+    time.sleep(2)
+    for s in socks.values():
+      messaging.drain_sock_raw(s)
+
+    st = time.monotonic()
+    ts = {s: list() for s in socks.keys()}
+    for _ in range(20):
+      for service, sock in socks.items():
+        for m in messaging.drain_sock(sock):
+          ts[service].append(m.logMonoTime)
+
+          # sanity check for corruption
+          assert m.valid
+          if service == "can":
+            assert len(m.can) == 0
+          elif service == "pandaStates":
+            assert len(m.pandaStates) == 1
+            ps = m.pandaStates[0]
+            assert ps.uptime < 100
+            assert ps.pandaType == "tres"
+            assert ps.ignitionLine
+            assert not ps.ignitionCan
+            assert ps.voltage < 14000
+          elif service == "peripheralState":
+            ps = m.peripheralState
+            assert ps.pandaType == "tres"
+            assert 4000 < ps.voltage < 14000
+            assert 100 < ps.current < 1000
+            assert ps.fanSpeedRpm < 8000
+
+      time.sleep(0.5)
+    et = time.monotonic() - st
+
+    print("\n======== timing report ========")
+    for service, times in ts.items():
+      dts = np.diff(times)/1e6
+      print(service.ljust(17), f"{np.mean(dts):7.2f} {np.min(dts):7.2f} {np.max(dts):7.2f}")
+      with subtests.test(msg="timing check", service=service):
+        edt = 1e3 / SERVICE_LIST[service].frequency
+        assert edt*0.9 < np.mean(dts) < edt*1.1
+        assert np.max(dts) < edt*3
+        assert np.min(dts) < edt
+        assert len(dts) >= ((et-0.5)*SERVICE_LIST[service].frequency*0.8)