From 3b7f740dd4883118747300bc3687074c2d3c2116 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Wed, 9 Aug 2023 18:36:11 -0700 Subject: [PATCH] draft --- selfdrive/car/car_helpers.py | 7 +- selfdrive/car/tests/test_models.py | 358 +++++++++++++++-------------- tools/lib/logreader.py | 5 + 3 files changed, 193 insertions(+), 177 deletions(-) diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py index 5bda64368b..1ad584de26 100644 --- a/selfdrive/car/car_helpers.py +++ b/selfdrive/car/car_helpers.py @@ -13,6 +13,8 @@ from system.swaglog import cloudlog import cereal.messaging as messaging from selfdrive.car import gen_empty_fingerprint +FRAME_FINGERPRINT = 100 # 1s + EventName = car.CarEvent.EventName @@ -126,7 +128,6 @@ def fingerprint(logcan, sendcan, num_pandas): finger = gen_empty_fingerprint() candidate_cars = {i: all_legacy_fingerprint_cars() for i in [0, 1]} # attempt fingerprint on both bus 0 and 1 frame = 0 - frame_fingerprint = 100 # 1s car_fingerprint = None done = False @@ -152,12 +153,12 @@ def fingerprint(logcan, sendcan, num_pandas): # if we only have one car choice and the time since we got our first # message has elapsed, exit for b in candidate_cars: - if len(candidate_cars[b]) == 1 and frame > frame_fingerprint: + if len(candidate_cars[b]) == 1 and frame > FRAME_FINGERPRINT: # fingerprint done car_fingerprint = candidate_cars[b][0] # bail if no cars left or we've been waiting for more than 2s - failed = (all(len(cc) == 0 for cc in candidate_cars.values()) and frame > frame_fingerprint) or frame > 200 + failed = (all(len(cc) == 0 for cc in candidate_cars.values()) and frame > FRAME_FINGERPRINT) or frame > 200 succeeded = car_fingerprint is not None done = failed or succeeded diff --git a/selfdrive/car/tests/test_models.py b/selfdrive/car/tests/test_models.py index 759cb51ad5..b418305ea0 100755 --- a/selfdrive/car/tests/test_models.py +++ b/selfdrive/car/tests/test_models.py @@ -2,6 +2,7 @@ # pylint: disable=E1101 import os import importlib +import time import unittest from collections import defaultdict, Counter from typing import List, Optional, Tuple @@ -11,7 +12,7 @@ from cereal import log, car from common.basedir import BASEDIR from common.realtime import DT_CTRL from selfdrive.car.fingerprints import all_known_cars -from selfdrive.car.car_helpers import interfaces +from selfdrive.car.car_helpers import FRAME_FINGERPRINT, interfaces from selfdrive.car.gm.values import CAR as GM from selfdrive.car.honda.values import CAR as HONDA, HONDA_BOSCH from selfdrive.car.hyundai.values import CAR as HYUNDAI @@ -19,6 +20,7 @@ from selfdrive.car.tests.routes import non_tested_cars, routes, CarTestRoute from selfdrive.test.openpilotci import get_url from tools.lib.logreader import LogReader from tools.lib.route import Route, SegmentName, RouteName +from selfdrive.car import gen_empty_fingerprint from panda.tests.libpanda import libpanda_py @@ -101,15 +103,22 @@ class TestCarModelBase(unittest.TestCase): except Exception: continue + t = time.perf_counter() car_fw = [] can_msgs = [] fingerprint = defaultdict(dict) - for msg in lr: + start_time = None + # fingerprint = gen_empty_fingerprint() + for idx, msg in enumerate(lr): if msg.which() == "can": + # gather fingerprint for 2s + can_msgs.append(msg) + if idx > FRAME_FINGERPRINT * 2: + continue for m in msg.can: if m.src < 64: fingerprint[m.src][m.address] = len(m.dat) - can_msgs.append(msg) + elif msg.which() == "carParams": car_fw = msg.carParams.carFw if msg.carParams.openpilotLongitudinalControl: @@ -118,6 +127,7 @@ class TestCarModelBase(unittest.TestCase): cls.car_model = msg.carParams.carFingerprint if len(can_msgs) > int(50 / DT_CTRL): + print('can_msgs', time.perf_counter() - t) break else: raise Exception(f"Route: {repr(cls.test_route.route)} with segments: {test_segs} not found or no CAN msgs found. Is it uploaded?") @@ -163,177 +173,177 @@ class TestCarModelBase(unittest.TestCase): else: raise Exception("unknown tuning") - def test_car_interface(self): - # TODO: also check for checksum violations from can parser - can_invalid_cnt = 0 - can_valid = False - CC = car.CarControl.new_message() - - for i, msg in enumerate(self.can_msgs): - CS = self.CI.update(CC, (msg.as_builder().to_bytes(),)) - self.CI.apply(CC, msg.logMonoTime) - - if CS.canValid: - can_valid = True - - # wait max of 2s for low frequency msgs to be seen - if i > 200 or can_valid: - can_invalid_cnt += not CS.canValid - - self.assertEqual(can_invalid_cnt, 0) - - def test_radar_interface(self): - os.environ['NO_RADAR_SLEEP'] = "1" - RadarInterface = importlib.import_module(f'selfdrive.car.{self.CP.carName}.radar_interface').RadarInterface - RI = RadarInterface(self.CP) - assert RI - - error_cnt = 0 - for i, msg in enumerate(self.can_msgs): - rr = RI.update((msg.as_builder().to_bytes(),)) - if rr is not None and i > 50: - error_cnt += car.RadarData.Error.canError in rr.errors - self.assertEqual(error_cnt, 0) - - def test_panda_safety_rx_valid(self): - if self.CP.dashcamOnly: - self.skipTest("no need to check panda safety for dashcamOnly") - - start_ts = self.can_msgs[0].logMonoTime - - failed_addrs = Counter() - for can in self.can_msgs: - # update panda timer - t = (can.logMonoTime - start_ts) / 1e3 - self.safety.set_timer(int(t)) - - # run all msgs through the safety RX hook - for msg in can.can: - if msg.src >= 64: - continue - - to_send = libpanda_py.make_CANPacket(msg.address, msg.src % 4, msg.dat) - if self.safety.safety_rx_hook(to_send) != 1: - failed_addrs[hex(msg.address)] += 1 - - # ensure all msgs defined in the addr checks are valid - if self.car_model not in ignore_addr_checks_valid: - self.safety.safety_tick_current_rx_checks() - if t > 1e6: - self.assertTrue(self.safety.addr_checks_valid()) - self.assertFalse(len(failed_addrs), f"panda safety RX check failed: {failed_addrs}") - - def test_panda_safety_tx_cases(self, data=None): - """Asserts we can tx common messages""" - if self.CP.notCar: - self.skipTest("Skipping test for notCar") - - def test_car_controller(car_control): - now_nanos = 0 - msgs_sent = 0 - CI = self.CarInterface(self.CP, self.CarController, self.CarState) - for _ in range(round(10.0 / DT_CTRL)): # make sure we hit the slowest messages - CI.update(car_control, []) - _, sendcan = CI.apply(car_control, now_nanos) - - now_nanos += DT_CTRL * 1e9 - msgs_sent += len(sendcan) - for addr, _, dat, bus in sendcan: - to_send = libpanda_py.make_CANPacket(addr, bus % 4, dat) - self.assertTrue(self.safety.safety_tx_hook(to_send), (addr, dat, bus)) - - # Make sure we attempted to send messages - self.assertGreater(msgs_sent, 50) - - # Make sure we can send all messages while inactive - CC = car.CarControl.new_message() - test_car_controller(CC) - - # Test cancel + general messages (controls_allowed=False & cruise_engaged=True) - self.safety.set_cruise_engaged_prev(True) - CC = car.CarControl.new_message(cruiseControl={'cancel': True}) - test_car_controller(CC) - - # Test resume + general messages (controls_allowed=True & cruise_engaged=True) - self.safety.set_controls_allowed(True) - CC = car.CarControl.new_message(cruiseControl={'resume': True}) - test_car_controller(CC) - - def test_panda_safety_carstate(self): - """ - Assert that panda safety matches openpilot's carState - """ - if self.CP.dashcamOnly: - self.skipTest("no need to check panda safety for dashcamOnly") - - CC = car.CarControl.new_message() - - # warm up pass, as initial states may be different - for can in self.can_msgs[:300]: - self.CI.update(CC, (can.as_builder().to_bytes(), )) - for msg in filter(lambda m: m.src in range(64), can.can): - to_send = libpanda_py.make_CANPacket(msg.address, msg.src % 4, msg.dat) - self.safety.safety_rx_hook(to_send) - - controls_allowed_prev = False - CS_prev = car.CarState.new_message() - checks = defaultdict(lambda: 0) - for idx, can in enumerate(self.can_msgs): - CS = self.CI.update(CC, (can.as_builder().to_bytes(), )) - for msg in filter(lambda m: m.src in range(64), can.can): - to_send = libpanda_py.make_CANPacket(msg.address, msg.src % 4, msg.dat) - ret = self.safety.safety_rx_hook(to_send) - self.assertEqual(1, ret, f"safety rx failed ({ret=}): {to_send}") - - # Skip first frame so CS_prev is properly initialized - if idx == 0: - CS_prev = CS - # Button may be left pressed in warm up period - if not self.CP.pcmCruise: - self.safety.set_controls_allowed(0) - continue - - # TODO: check rest of panda's carstate (steering, ACC main on, etc.) - - checks['gasPressed'] += CS.gasPressed != self.safety.get_gas_pressed_prev() - if self.CP.carName not in ("hyundai", "body"): - # TODO: fix standstill mismatches for other makes - checks['standstill'] += CS.standstill == self.safety.get_vehicle_moving() - - # TODO: remove this exception once this mismatch is resolved - brake_pressed = CS.brakePressed - if CS.brakePressed and not self.safety.get_brake_pressed_prev(): - if self.CP.carFingerprint in (HONDA.PILOT, HONDA.RIDGELINE) and CS.brake > 0.05: - brake_pressed = False - checks['brakePressed'] += brake_pressed != self.safety.get_brake_pressed_prev() - checks['regenBraking'] += CS.regenBraking != self.safety.get_regen_braking_prev() - - if self.CP.pcmCruise: - # On most pcmCruise cars, openpilot's state is always tied to the PCM's cruise state. - # On Honda Nidec, we always engage on the rising edge of the PCM cruise state, but - # openpilot brakes to zero even if the min ACC speed is non-zero (i.e. the PCM disengages). - if self.CP.carName == "honda" and self.CP.carFingerprint not in HONDA_BOSCH: - # only the rising edges are expected to match - if CS.cruiseState.enabled and not CS_prev.cruiseState.enabled: - checks['controlsAllowed'] += not self.safety.get_controls_allowed() - else: - checks['controlsAllowed'] += not CS.cruiseState.enabled and self.safety.get_controls_allowed() - else: - # Check for enable events on rising edge of controls allowed - button_enable = any(evt.enable for evt in CS.events) - mismatch = button_enable != (self.safety.get_controls_allowed() and not controls_allowed_prev) - checks['controlsAllowed'] += mismatch - controls_allowed_prev = self.safety.get_controls_allowed() - if button_enable and not mismatch: - self.safety.set_controls_allowed(False) - - if self.CP.carName == "honda": - checks['mainOn'] += CS.cruiseState.available != self.safety.get_acc_main_on() - - CS_prev = CS - - failed_checks = {k: v for k, v in checks.items() if v > 0} - self.assertFalse(len(failed_checks), f"panda safety doesn't agree with openpilot: {failed_checks}") + # def test_car_interface(self): + # # TODO: also check for checksum violations from can parser + # can_invalid_cnt = 0 + # can_valid = False + # CC = car.CarControl.new_message() + # + # for i, msg in enumerate(self.can_msgs): + # CS = self.CI.update(CC, (msg.as_builder().to_bytes(),)) + # self.CI.apply(CC, msg.logMonoTime) + # + # if CS.canValid: + # can_valid = True + # + # # wait max of 2s for low frequency msgs to be seen + # if i > 200 or can_valid: + # can_invalid_cnt += not CS.canValid + # + # self.assertEqual(can_invalid_cnt, 0) + # + # def test_radar_interface(self): + # os.environ['NO_RADAR_SLEEP'] = "1" + # RadarInterface = importlib.import_module(f'selfdrive.car.{self.CP.carName}.radar_interface').RadarInterface + # RI = RadarInterface(self.CP) + # assert RI + # + # error_cnt = 0 + # for i, msg in enumerate(self.can_msgs): + # rr = RI.update((msg.as_builder().to_bytes(),)) + # if rr is not None and i > 50: + # error_cnt += car.RadarData.Error.canError in rr.errors + # self.assertEqual(error_cnt, 0) + # + # def test_panda_safety_rx_valid(self): + # if self.CP.dashcamOnly: + # self.skipTest("no need to check panda safety for dashcamOnly") + # + # start_ts = self.can_msgs[0].logMonoTime + # + # failed_addrs = Counter() + # for can in self.can_msgs: + # # update panda timer + # t = (can.logMonoTime - start_ts) / 1e3 + # self.safety.set_timer(int(t)) + # + # # run all msgs through the safety RX hook + # for msg in can.can: + # if msg.src >= 64: + # continue + # + # to_send = libpanda_py.make_CANPacket(msg.address, msg.src % 4, msg.dat) + # if self.safety.safety_rx_hook(to_send) != 1: + # failed_addrs[hex(msg.address)] += 1 + # + # # ensure all msgs defined in the addr checks are valid + # if self.car_model not in ignore_addr_checks_valid: + # self.safety.safety_tick_current_rx_checks() + # if t > 1e6: + # self.assertTrue(self.safety.addr_checks_valid()) + # self.assertFalse(len(failed_addrs), f"panda safety RX check failed: {failed_addrs}") + + # def test_panda_safety_tx_cases(self, data=None): + # """Asserts we can tx common messages""" + # if self.CP.notCar: + # self.skipTest("Skipping test for notCar") + # + # def test_car_controller(car_control): + # now_nanos = 0 + # msgs_sent = 0 + # CI = self.CarInterface(self.CP, self.CarController, self.CarState) + # for _ in range(round(10.0 / DT_CTRL)): # make sure we hit the slowest messages + # CI.update(car_control, []) + # _, sendcan = CI.apply(car_control, now_nanos) + # + # now_nanos += DT_CTRL * 1e9 + # msgs_sent += len(sendcan) + # for addr, _, dat, bus in sendcan: + # to_send = libpanda_py.make_CANPacket(addr, bus % 4, dat) + # self.assertTrue(self.safety.safety_tx_hook(to_send), (addr, dat, bus)) + # + # # Make sure we attempted to send messages + # self.assertGreater(msgs_sent, 50) + # + # # Make sure we can send all messages while inactive + # CC = car.CarControl.new_message() + # test_car_controller(CC) + # + # # Test cancel + general messages (controls_allowed=False & cruise_engaged=True) + # self.safety.set_cruise_engaged_prev(True) + # CC = car.CarControl.new_message(cruiseControl={'cancel': True}) + # test_car_controller(CC) + # + # # Test resume + general messages (controls_allowed=True & cruise_engaged=True) + # self.safety.set_controls_allowed(True) + # CC = car.CarControl.new_message(cruiseControl={'resume': True}) + # test_car_controller(CC) + + # def test_panda_safety_carstate(self): + # """ + # Assert that panda safety matches openpilot's carState + # """ + # if self.CP.dashcamOnly: + # self.skipTest("no need to check panda safety for dashcamOnly") + # + # CC = car.CarControl.new_message() + # + # # warm up pass, as initial states may be different + # for can in self.can_msgs[:300]: + # self.CI.update(CC, (can.as_builder().to_bytes(), )) + # for msg in filter(lambda m: m.src in range(64), can.can): + # to_send = libpanda_py.make_CANPacket(msg.address, msg.src % 4, msg.dat) + # self.safety.safety_rx_hook(to_send) + # + # controls_allowed_prev = False + # CS_prev = car.CarState.new_message() + # checks = defaultdict(lambda: 0) + # for idx, can in enumerate(self.can_msgs): + # CS = self.CI.update(CC, (can.as_builder().to_bytes(), )) + # for msg in filter(lambda m: m.src in range(64), can.can): + # to_send = libpanda_py.make_CANPacket(msg.address, msg.src % 4, msg.dat) + # ret = self.safety.safety_rx_hook(to_send) + # self.assertEqual(1, ret, f"safety rx failed ({ret=}): {to_send}") + # + # # Skip first frame so CS_prev is properly initialized + # if idx == 0: + # CS_prev = CS + # # Button may be left pressed in warm up period + # if not self.CP.pcmCruise: + # self.safety.set_controls_allowed(0) + # continue + # + # # TODO: check rest of panda's carstate (steering, ACC main on, etc.) + # + # checks['gasPressed'] += CS.gasPressed != self.safety.get_gas_pressed_prev() + # if self.CP.carName not in ("hyundai", "body"): + # # TODO: fix standstill mismatches for other makes + # checks['standstill'] += CS.standstill == self.safety.get_vehicle_moving() + # + # # TODO: remove this exception once this mismatch is resolved + # brake_pressed = CS.brakePressed + # if CS.brakePressed and not self.safety.get_brake_pressed_prev(): + # if self.CP.carFingerprint in (HONDA.PILOT, HONDA.RIDGELINE) and CS.brake > 0.05: + # brake_pressed = False + # checks['brakePressed'] += brake_pressed != self.safety.get_brake_pressed_prev() + # checks['regenBraking'] += CS.regenBraking != self.safety.get_regen_braking_prev() + # + # if self.CP.pcmCruise: + # # On most pcmCruise cars, openpilot's state is always tied to the PCM's cruise state. + # # On Honda Nidec, we always engage on the rising edge of the PCM cruise state, but + # # openpilot brakes to zero even if the min ACC speed is non-zero (i.e. the PCM disengages). + # if self.CP.carName == "honda" and self.CP.carFingerprint not in HONDA_BOSCH: + # # only the rising edges are expected to match + # if CS.cruiseState.enabled and not CS_prev.cruiseState.enabled: + # checks['controlsAllowed'] += not self.safety.get_controls_allowed() + # else: + # checks['controlsAllowed'] += not CS.cruiseState.enabled and self.safety.get_controls_allowed() + # else: + # # Check for enable events on rising edge of controls allowed + # button_enable = any(evt.enable for evt in CS.events) + # mismatch = button_enable != (self.safety.get_controls_allowed() and not controls_allowed_prev) + # checks['controlsAllowed'] += mismatch + # controls_allowed_prev = self.safety.get_controls_allowed() + # if button_enable and not mismatch: + # self.safety.set_controls_allowed(False) + # + # if self.CP.carName == "honda": + # checks['mainOn'] += CS.cruiseState.available != self.safety.get_acc_main_on() + # + # CS_prev = CS + # + # failed_checks = {k: v for k, v in checks.items() if v > 0} + # self.assertFalse(len(failed_checks), f"panda safety doesn't agree with openpilot: {failed_checks}") @parameterized_class(('car_model', 'test_route'), test_cases) diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index 46c648ef12..9eafa28a7f 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -2,6 +2,7 @@ import os import sys import bz2 +import time import urllib.parse import capnp import warnings @@ -85,11 +86,15 @@ class LogReader: # old rlogs weren't bz2 compressed raise Exception(f"unknown extension {ext}") + t = time.perf_counter() with FileReader(fn) as f: dat = f.read() + print('download time', time.perf_counter() - t) + t = time.perf_counter() if ext == ".bz2" or dat.startswith(b'BZh9'): dat = bz2.decompress(dat) + print('decomp time', time.perf_counter() - t) ents = capnp_log.Event.read_multiple_bytes(dat)