Shane Smiskol
1 year ago
parent
3b52f1f5d8
commit
dfe8d0685c
3 changed files with 378 additions and 5 deletions
@ -0,0 +1,355 @@ |
||||
#!/usr/bin/env python3 |
||||
import capnp |
||||
import os |
||||
import importlib |
||||
import pytest |
||||
import random |
||||
import unittest |
||||
from collections import defaultdict, Counter |
||||
from typing import List, Optional, Tuple |
||||
from parameterized import parameterized_class |
||||
import hypothesis.strategies as st |
||||
from hypothesis.stateful import RuleBasedStateMachine, rule, precondition |
||||
from hypothesis import HealthCheck, Phase, assume, given, settings |
||||
|
||||
from cereal import messaging, log, car |
||||
from openpilot.common.basedir import BASEDIR |
||||
from openpilot.common.params import Params |
||||
from openpilot.common.realtime import DT_CTRL |
||||
from openpilot.selfdrive.car.fingerprints import all_known_cars |
||||
from openpilot.selfdrive.car.car_helpers import FRAME_FINGERPRINT, interfaces |
||||
from openpilot.selfdrive.car.gm.values import CAR as GM |
||||
from openpilot.selfdrive.car.honda.values import CAR as HONDA, HONDA_BOSCH |
||||
from openpilot.selfdrive.car.hyundai.values import CAR as HYUNDAI |
||||
from openpilot.selfdrive.car.tests.routes import non_tested_cars, routes, CarTestRoute |
||||
from openpilot.selfdrive.controls.controlsd import Controls |
||||
from openpilot.selfdrive.test.openpilotci import get_url |
||||
from openpilot.tools.lib.logreader import LogReader |
||||
from openpilot.tools.lib.route import Route, SegmentName, RouteName |
||||
|
||||
from panda.tests.libpanda import libpanda_py |
||||
|
||||
EventName = car.CarEvent.EventName |
||||
PandaType = log.PandaState.PandaType |
||||
SafetyModel = car.CarParams.SafetyModel |
||||
|
||||
NUM_JOBS = int(os.environ.get("NUM_JOBS", "1")) |
||||
JOB_ID = int(os.environ.get("JOB_ID", "0")) |
||||
INTERNAL_SEG_LIST = os.environ.get("INTERNAL_SEG_LIST", "") |
||||
INTERNAL_SEG_CNT = int(os.environ.get("INTERNAL_SEG_CNT", "0")) |
||||
|
||||
ignore_addr_checks_valid = [ |
||||
GM.BUICK_REGAL, |
||||
HYUNDAI.GENESIS_G70_2020, |
||||
] |
||||
|
||||
|
||||
def get_test_cases() -> List[Tuple[str, Optional[CarTestRoute]]]: |
||||
# build list of test cases |
||||
test_cases = [] |
||||
if not len(INTERNAL_SEG_LIST): |
||||
routes_by_car = defaultdict(set) |
||||
for r in routes: |
||||
routes_by_car[r.car_model].add(r) |
||||
|
||||
for i, c in enumerate(sorted(all_known_cars())): |
||||
if i % NUM_JOBS == JOB_ID: |
||||
test_cases.extend(sorted((c, r) for r in routes_by_car.get(c, (None,)))) |
||||
|
||||
else: |
||||
with open(os.path.join(BASEDIR, INTERNAL_SEG_LIST), "r") as f: |
||||
seg_list = f.read().splitlines() |
||||
|
||||
cnt = INTERNAL_SEG_CNT or len(seg_list) |
||||
seg_list_iter = iter(seg_list[:cnt]) |
||||
|
||||
for platform in seg_list_iter: |
||||
platform = platform[2:] # get rid of comment |
||||
segment_name = SegmentName(next(seg_list_iter)) |
||||
test_cases.append((platform, CarTestRoute(segment_name.route_name.canonical_name, platform, |
||||
segment=segment_name.segment_num))) |
||||
return test_cases |
||||
|
||||
|
||||
init_done = False |
||||
|
||||
@pytest.mark.slow |
||||
class CarModelBase(RuleBasedStateMachine): |
||||
car_model: Optional[str] = 'TOYOTA CAMRY 2021' |
||||
test_route: Optional[CarTestRoute] = CarTestRoute("3456ad0cd7281b24|2020-12-13--17-45-56", 'TOYOTA CAMRY 2021') |
||||
ci: bool = True |
||||
|
||||
can_msgs: List[capnp.lib.capnp._DynamicStructReader] |
||||
elm_frame: Optional[int] |
||||
|
||||
def __init__(self): |
||||
super().__init__() |
||||
global init_done |
||||
if init_done: |
||||
return |
||||
init_done = True |
||||
# if self.__name__ == 'TestCarModel' or self.__name__.endswith('Base'): |
||||
# raise unittest.SkipTest |
||||
|
||||
if 'FILTER' in os.environ: |
||||
if not self.car_model.startswith(tuple(os.environ.get('FILTER').split(','))): |
||||
raise unittest.SkipTest |
||||
|
||||
if self.test_route is None: |
||||
if self.car_model in non_tested_cars: |
||||
print(f"Skipping tests for {self.car_model}: missing route") |
||||
raise unittest.SkipTest |
||||
raise Exception(f"missing test route for {self.car_model}") |
||||
|
||||
test_segs = (2, 1, 0) |
||||
if self.test_route.segment is not None: |
||||
test_segs = (self.test_route.segment,) |
||||
|
||||
for seg in test_segs: |
||||
try: |
||||
if len(INTERNAL_SEG_LIST): |
||||
route_name = RouteName(self.test_route.route) |
||||
lr = LogReader(f"cd:/{route_name.dongle_id}/{route_name.time_str}/{seg}/rlog.bz2") |
||||
elif self.ci: |
||||
lr = LogReader(get_url(self.test_route.route, seg)) |
||||
else: |
||||
lr = LogReader(Route(self.test_route.route).log_paths()[seg]) |
||||
except Exception: |
||||
continue |
||||
|
||||
car_fw = [] |
||||
can_msgs = [] |
||||
self.elm_frame = None |
||||
fingerprint = defaultdict(dict) |
||||
experimental_long = False |
||||
enabled_toggle = True |
||||
dashcam_only = False |
||||
for msg in lr: |
||||
if msg.which() == "can": |
||||
can_msgs.append(msg) |
||||
if len(can_msgs) <= FRAME_FINGERPRINT: |
||||
for m in msg.can: |
||||
if m.src < 64: |
||||
fingerprint[m.src][m.address] = len(m.dat) |
||||
|
||||
elif msg.which() == "carParams": |
||||
car_fw = msg.carParams.carFw |
||||
dashcam_only = msg.carParams.dashcamOnly |
||||
if msg.carParams.openpilotLongitudinalControl: |
||||
experimental_long = True |
||||
if self.car_model is None and not self.ci: |
||||
self.car_model = msg.carParams.carFingerprint |
||||
|
||||
elif msg.which() == 'initData': |
||||
for param in msg.initData.params.entries: |
||||
if param.key == 'OpenpilotEnabledToggle': |
||||
enabled_toggle = param.value.strip(b'\x00') == b'1' |
||||
|
||||
# Log which can frame the panda safety mode left ELM327, for CAN validity checks |
||||
if msg.which() == 'pandaStates': |
||||
for ps in msg.pandaStates: |
||||
if self.elm_frame is None and ps.safetyModel != SafetyModel.elm327: |
||||
self.elm_frame = len(can_msgs) |
||||
|
||||
elif msg.which() == 'pandaStateDEPRECATED': |
||||
if self.elm_frame is None and msg.pandaStateDEPRECATED.safetyModel != SafetyModel.elm327: |
||||
self.elm_frame = len(can_msgs) |
||||
|
||||
if len(can_msgs) > int(50 / DT_CTRL): |
||||
break |
||||
else: |
||||
raise Exception(f"Route: {repr(self.test_route.route)} with segments: {test_segs} not found or no CAN msgs found. Is it uploaded?") |
||||
|
||||
# if relay is expected to be open in the route |
||||
self.openpilot_enabled = enabled_toggle and not dashcam_only |
||||
|
||||
self.can_msgs = sorted(can_msgs, key=lambda msg: msg.logMonoTime) |
||||
|
||||
self.CarInterface, self.CarController, self.CarState = interfaces[self.car_model] |
||||
self.CP = self.CarInterface.get_params(self.car_model, fingerprint, car_fw, experimental_long, docs=False) |
||||
assert self.CP |
||||
assert self.CP.carFingerprint == self.car_model |
||||
|
||||
self.car_state_dict = {'gas_pressed': False} |
||||
print("HERE!!!!!") |
||||
|
||||
# print('SETUP HEREHEREHEREHEREHEREHEREHEREHEREHEREHEREHEREHEREHEREHEREHEREHEREHERE') |
||||
self.CI = self.CarInterface(self.CP.copy(), self.CarController, self.CarState) |
||||
assert self.CI |
||||
|
||||
Params().put_bool("OpenpilotEnabledToggle", self.openpilot_enabled) |
||||
|
||||
# TODO: check safetyModel is in release panda build |
||||
self.safety = libpanda_py.libpanda |
||||
|
||||
cfg = self.CP.safetyConfigs[-1] |
||||
set_status = self.safety.set_safety_hooks(cfg.safetyModel.raw, cfg.safetyParam) |
||||
assert 0 == set_status, f"failed to set safetyModel {cfg}" |
||||
self.safety.init_tests() |
||||
|
||||
@rule(messages=st.lists(st.tuples( |
||||
st.integers(min_value=0, max_value=2), |
||||
st.integers(min_value=0, max_value=0x1FFFF), |
||||
st.binary(min_size=8, max_size=8) |
||||
))) |
||||
def send_messages(self, messages): |
||||
print(messages) |
||||
for bus, address, dat in messages: |
||||
to_send = libpanda_py.make_CANPacket(address, bus, dat) |
||||
self.safety.safety_rx_hook(to_send) |
||||
|
||||
can = messaging.new_message('can', 1) |
||||
can.can = [log.CanData(address=address, dat=dat, src=bus)] |
||||
|
||||
CC = car.CarControl.new_message() |
||||
prev_car_state = self.car_state.copy() |
||||
self.car_state = self.CI.update(CC, (can.to_bytes(),)) |
||||
|
||||
# Check for state change. |
||||
self.check_state_change(prev_car_state, self.car_state) |
||||
|
||||
# Only start asserting once there's been a state change. |
||||
if self.has_state_changed: |
||||
# Assertions based on your criteria here |
||||
# For example: |
||||
assert self.car_state['gas_pressed'] == self.initial_car_state['gas_pressed'], \ |
||||
"Mismatch in gas pressed state." |
||||
|
||||
# @settings(max_examples=100, deadline=None, |
||||
# # phases=(Phase.reuse, Phase.generate, Phase.shrink), |
||||
# suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], |
||||
# ) |
||||
# @given(data=st.data()) |
||||
# def test_panda_safety_carstate_fuzzy(self, data): |
||||
# state_has_changed = lambda prev_state, new_state: prev_state != new_state |
||||
# # cfg = self.CP.safetyConfigs[-1] |
||||
# # set_status = self.safety.set_safety_hooks(cfg.safetyModel.raw, cfg.safetyParam) |
||||
# # self.assertEqual(0, set_status, f"failed to set safetyModel {cfg}") |
||||
# # self.safety.init_tests() |
||||
# |
||||
# # bus = 0 # random.randint(0, 3) |
||||
# # address = 0xaa # random.randint(0x200, 0x300) |
||||
# |
||||
# address = data.draw(st.integers(0x1ff, 0x250)) |
||||
# bus = 0 |
||||
# |
||||
# # ORIG: |
||||
# # msg_strategy = st.tuples(st.integers(min_value=0, max_value=0), st.integers(min_value=0x100, max_value=0x400), st.binary(min_size=8, max_size=8)) |
||||
# |
||||
# msg_strategy = st.binary(min_size=8, max_size=8) |
||||
# msgs = data.draw(st.lists(msg_strategy, min_size=100))#, min_size=100, max_size=1000)) |
||||
# print(len(msgs)) |
||||
# |
||||
# start_gas = self.safety.get_gas_pressed_prev() |
||||
# start_gas_int_detected = self.safety.get_gas_interceptor_detected() |
||||
# |
||||
# # for bus, address, dat in msgs: |
||||
# for dat in msgs: |
||||
# to_send = libpanda_py.make_CANPacket(address, bus, dat) |
||||
# did_rx = self.safety.safety_rx_hook(to_send) |
||||
# |
||||
# can = messaging.new_message('can', 1) |
||||
# can.can = [log.CanData(address=address, dat=dat, src=bus)] |
||||
# |
||||
# CC = car.CarControl.new_message() |
||||
# CS = self.CI.update(CC, (can.to_bytes(),)) |
||||
# |
||||
# if self.safety.get_gas_interceptor_detected():# and state_has_changed(start_gas, self.safety.get_gas_pressed_prev()): |
||||
# print('get_gas_interceptor_detected!') |
||||
# # self.assertEqual(CS.gasPressed, self.safety.get_gas_interceptor_prev()) |
||||
# self.assertEqual(CS.gasPressed, self.safety.get_gas_pressed_prev()) |
||||
# # self.assertFalse(True) |
||||
# |
||||
# |
||||
# # if self.safety.get_gas_pressed_prev() and self.safety.get_cruise_engaged_prev(): |
||||
# # self.assertFalse(True) |
||||
# # self.assertFalse(self.safety.get_cruise_engaged_prev()) |
||||
# |
||||
# # print('gas_pressed', CS.gasPressed, self.safety.get_gas_pressed_prev()) |
||||
# # print('wheel_speeds', CS.wheelSpeeds) |
||||
# # print('standstill', CS.standstill, not self.safety.get_vehicle_moving()) |
||||
# |
||||
# # print('did_rx', did_rx) |
||||
# # if did_rx: |
||||
# # self.assertFalse(True, 'finally did rx: {}, {}'.format(i, dat)) |
||||
# # self.assertTrue(CS.standstill, (not CS.standstill, self.safety.get_vehicle_moving(), CS.vEgoRaw, CS.wheelSpeeds)) |
||||
# |
||||
# |
||||
# # self.assertEqual(CS.gasPressed, self.safety.get_gas_pressed_prev()) |
||||
# # self.assertEqual(not CS.standstill, self.safety.get_vehicle_moving()) |
||||
# # self.assertEqual(CS.brakePressed, self.safety.get_brake_pressed_prev()) |
||||
# # self.assertEqual(CS.regenBraking, self.safety.get_regen_braking_prev()) |
||||
# # |
||||
# # if self.CP.pcmCruise: |
||||
# # self.assertEqual(CS.cruiseState.enabled, self.safety.get_cruise_engaged_prev()) |
||||
# # |
||||
# # if self.CP.carName == "honda": |
||||
# # self.assertEqual(CS.cruiseState.available, self.safety.get_acc_main_on()) |
||||
# |
||||
# |
||||
# # if self.safety.get_gas_interceptor_detected(): |
||||
# # print('get_gas_interceptor_detected!') |
||||
# # # self.assertEqual(CS.gasPressed, self.safety.get_gas_interceptor_prev()) |
||||
# # self.assertEqual(CS.gasPressed, self.safety.get_gas_pressed_prev()) |
||||
# # # self.assertFalse(True) |
||||
# |
||||
# print(self.safety.get_gas_pressed_prev(), self.safety.get_brake_pressed_prev(), self.safety.get_vehicle_moving(), self.safety.get_cruise_engaged_prev()) |
||||
# assume(state_has_changed(False, self.safety.get_gas_pressed_prev())) |
||||
# # assume(state_has_changed(start_gas, self.safety.get_gas_pressed_prev())) # this just goes on forever |
||||
# # assume(state_has_changed(start_gas_int_detected, self.safety.get_gas_interceptor_detected())) |
||||
# # assume(state_has_changed(False, self.safety.get_brake_pressed_prev())) |
||||
# # assume(state_has_changed(False, self.safety.get_vehicle_moving())) |
||||
# # assume(state_has_changed(False, self.safety.get_cruise_engaged_prev())) |
||||
# |
||||
# # print(msgs) |
||||
# # print('\nresults', self.safety.get_gas_pressed_prev(), self.safety.get_vehicle_moving(), self.safety.get_brake_pressed_prev(), self.safety.get_regen_braking_prev(), self.safety.get_cruise_engaged_prev(), self.safety.get_acc_main_on()) |
||||
# return |
||||
# |
||||
# for i in range(1000): |
||||
# # self.setUp() |
||||
# dat = os.urandom(8) |
||||
# to_send = libpanda_py.make_CANPacket(address, bus, dat) |
||||
# did_rx = self.safety.safety_rx_hook(to_send) |
||||
# |
||||
# can = messaging.new_message('can', 1) |
||||
# can.can = [log.CanData(address=address, dat=dat, src=bus)] |
||||
# |
||||
# CC = car.CarControl.new_message() |
||||
# CS = self.CI.update(CC, (can.to_bytes(), )) |
||||
# |
||||
# print('gas_pressed', CS.gasPressed, self.safety.get_gas_pressed_prev()) |
||||
# print('wheel_speeds', CS.wheelSpeeds) |
||||
# print('standstill', CS.standstill, not self.safety.get_vehicle_moving()) |
||||
# |
||||
# print('did_rx', did_rx) |
||||
# # if did_rx: |
||||
# # self.assertFalse(True, 'finally did rx: {}, {}'.format(i, dat)) |
||||
# self.assertEqual(not CS.standstill, self.safety.get_vehicle_moving()) |
||||
# |
||||
# print('\nresults', self.safety.get_gas_pressed_prev(), self.safety.get_vehicle_moving(), self.safety.get_brake_pressed_prev(), self.safety.get_regen_braking_prev(), self.safety.get_cruise_engaged_prev(), self.safety.get_acc_main_on()) |
||||
# |
||||
# # self.assertEqual(CS.gasPressed, self.safety.get_gas_pressed_prev()) |
||||
# # self.assertEqual(not CS.standstill, self.safety.get_vehicle_moving()) |
||||
# # self.assertEqual(CS.brakePressed, self.safety.get_brake_pressed_prev()) |
||||
# # self.assertEqual(CS.regenBraking, self.safety.get_regen_braking_prev()) |
||||
# # |
||||
# # if self.CP.pcmCruise: |
||||
# # self.assertEqual(CS.cruiseState.enabled, self.safety.get_cruise_engaged_prev()) |
||||
# # |
||||
# # if self.CP.carName == "honda": |
||||
# # self.assertEqual(CS.cruiseState.available, self.safety.get_acc_main_on()) |
||||
|
||||
|
||||
# @parameterized_class(('car_model', 'test_route'), get_test_cases()) |
||||
# class TestCarModel(TestCarModelBase): |
||||
# pass |
||||
|
||||
TestCarModelBase = CarModelBase.TestCase |
||||
|
||||
# TestCarModelBase().runTest() |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
unittest.main() |
||||
|
||||
|
||||
Loading…
Reference in new issue