test car models: fix all panda safety mismatches (#23616)

* remove tolerance

* fix nissan mismatches

* warm up

* remove additional bosch check

* another honda exception

* little more

* Update selfdrive/test/test_models.py
old-commit-hash: d2735d73fc

selfdrive/car/honda/carstate.py

@@ -240,14 +240,10 @@ class CarState(CarStateBase):
     gear = int(cp.vl[self.gearbox_msg]["GEAR_SHIFTER"])
     ret.gearShifter = self.parse_gear_shifter(self.shifter_values.get(gear, None))
 
-    ret.gas = cp.vl["POWERTRAIN_DATA"]["PEDAL_GAS"]
-
-    # this is a hack for the interceptor. This is now only used in the simulation
-    # TODO: Replace tests by toyota so this can go away
     if self.CP.enableGasInterceptor:
-      user_gas = (cp.vl["GAS_SENSOR"]["INTERCEPTOR_GAS"] + cp.vl["GAS_SENSOR"]["INTERCEPTOR_GAS2"]) / 2.
-      ret.gasPressed = user_gas > 1e-5  # this works because interceptor reads < 0 when pedal position is 0. Once calibrated, this will change
+      ret.gas = (cp.vl["GAS_SENSOR"]["INTERCEPTOR_GAS"] + cp.vl["GAS_SENSOR"]["INTERCEPTOR_GAS2"]) / 2.
     else:
+      ret.gas = cp.vl["POWERTRAIN_DATA"]["PEDAL_GAS"]
     ret.gasPressed = ret.gas > 1e-5
 
     ret.steeringTorque = cp.vl["STEER_STATUS"]["STEER_TORQUE_SENSOR"]

selfdrive/test/test_models.py

@@ -8,11 +8,13 @@ from parameterized import parameterized_class
 
 from cereal import log, car
 from common.params import Params
+from selfdrive.boardd.boardd import can_capnp_to_can_list, can_list_to_can_capnp
 from selfdrive.car.fingerprints import all_known_cars
 from selfdrive.car.car_helpers import interfaces
 from selfdrive.car.gm.values import CAR as GM
-from selfdrive.car.honda.values import HONDA_BOSCH, CAR as HONDA
+from selfdrive.car.honda.values import CAR as HONDA
 from selfdrive.car.hyundai.values import CAR as HYUNDAI
+from selfdrive.car.toyota.values import CAR as TOYOTA
 from selfdrive.test.test_routes import routes, non_tested_cars
 from selfdrive.test.openpilotci import get_url
 from tools.lib.logreader import LogReader
@@ -174,44 +176,56 @@ class TestCarModel(unittest.TestCase):
     if self.CP.dashcamOnly:
       self.skipTest("no need to check panda safety for dashcamOnly")
 
-    checks = defaultdict(lambda: 0)
     CC = car.CarControl.new_message()
+
+    # warm up pass, as initial states may be different
+    for can in self.can_msgs[:300]:
+      for msg in can_capnp_to_can_list(can.can, src_filter=range(64)):
+        to_send = package_can_msg(msg)
+        self.safety.safety_rx_hook(to_send)
+        self.CI.update(CC, (can_list_to_can_capnp([msg, ]), ))
+
+    checks = defaultdict(lambda: 0)
     for can in self.can_msgs:
-      for msg in can.can:
-        if msg.src >= 64:
-          continue
-        to_send = package_can_msg([msg.address, 0, msg.dat, msg.src])
+      for msg in can_capnp_to_can_list(can.can, src_filter=range(64)):
+        to_send = package_can_msg(msg)
         ret = self.safety.safety_rx_hook(to_send)
         self.assertEqual(1, ret, f"safety rx failed ({ret=}): {to_send}")
-      CS = self.CI.update(CC, (can.as_builder().to_bytes(),))
+        CS = self.CI.update(CC, (can_list_to_can_capnp([msg, ]), ))
+
+      # TODO: check rest of panda's carstate (steering, ACC main on, etc.)
+
+      # TODO: make the interceptor thresholds in openpilot and panda match, then remove this exception
+      gas_pressed = CS.gasPressed
+      if self.CP.enableGasInterceptor and gas_pressed and not self.safety.get_gas_pressed_prev():
+        # panda intentionally has a higher threshold
+        if self.CP.carName == "toyota" and 15 < CS.gas < 15*1.5:
+          gas_pressed = False
+        if self.CP.carName == "honda":
+          gas_pressed = False
+      checks['gasPressed'] += gas_pressed != self.safety.get_gas_pressed_prev()
+
+      # TODO: remove this exception once this mismatch is resolved
+      brake_pressed = CS.brakePressed
+      if CS.brakePressed and not self.safety.get_brake_pressed_prev():
+        if self.CP.carFingerprint in (HONDA.PILOT, HONDA.PASSPORT, HONDA.RIDGELINE) and CS.brake > 0.05:
+          brake_pressed = False
+        if self.CP.carName == "honda" and self.CI.CS.brake_switch:
+          brake_pressed = False
+      checks['brakePressed'] += brake_pressed != self.safety.get_brake_pressed_prev()
 
-      # TODO: check steering state
-      # check that openpilot and panda safety agree on the car's state
-      checks['gasPressed'] += CS.gasPressed != self.safety.get_gas_pressed_prev()
-      checks['brakePressed'] += CS.brakePressed != self.safety.get_brake_pressed_prev()
       if self.CP.pcmCruise:
         checks['controlsAllowed'] += not CS.cruiseState.enabled and self.safety.get_controls_allowed()
 
-      # TODO: extend this to all cars
       if self.CP.carName == "honda":
         checks['mainOn'] += CS.cruiseState.available != self.safety.get_acc_main_on()
 
-    # TODO: reduce tolerance to 0
-    failed_checks = {k: v for k, v in checks.items() if v > 25}
-
-    # TODO: the panda and openpilot interceptor thresholds should match
-    skip_gas_check = self.CP.carName == 'chrysler'
-    if "gasPressed" in failed_checks and (self.CP.enableGasInterceptor or skip_gas_check):
-      if failed_checks['gasPressed'] < 150 or skip_gas_check:
-        del failed_checks['gasPressed']
-
-    # TODO: honda nidec: do same checks in carState and panda
-    if "brakePressed" in failed_checks and self.CP.carName == 'honda' and \
-      (self.car_model not in HONDA_BOSCH or self.car_model in [HONDA.CRV_HYBRID, HONDA.HONDA_E]):
-      if failed_checks['brakePressed'] < 150:
-        del failed_checks['brakePressed']
+    # TODO: add flag to toyota safety
+    if self.CP.carFingerprint == TOYOTA.SIENNA and checks['brakePressed'] < 25:
+      checks['brakePressed'] = 0
 
-    self.assertFalse(len(failed_checks), f"panda safety doesn't agree with CarState: {failed_checks}")
+    failed_checks = {k: v for k, v in checks.items() if v > 0}
+    self.assertFalse(len(failed_checks), f"panda safety doesn't agree with openpilot: {failed_checks}")
 
 if __name__ == "__main__":
   unittest.main()