From c8429f913e15934cf46564f23584cc454699ffdc Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 30 Jan 2022 11:17:51 -0800 Subject: [PATCH] test car models improvements (#23668) * run on all routes * min segment length * invalid * revert that --- selfdrive/test/test_models.py | 30 ++++++++---- selfdrive/test/test_routes.py | 21 +-------- .../layouts/controls_mismatch_debug.xml | 46 +++++++++++++++++++ 3 files changed, 67 insertions(+), 30 deletions(-) create mode 100644 tools/plotjuggler/layouts/controls_mismatch_debug.xml diff --git a/selfdrive/test/test_models.py b/selfdrive/test/test_models.py index 93312cc12b..f8f6da6d7a 100755 --- a/selfdrive/test/test_models.py +++ b/selfdrive/test/test_models.py @@ -4,10 +4,12 @@ import os import importlib import unittest from collections import defaultdict, Counter +from typing import List, Optional, Tuple from parameterized import parameterized_class from cereal import log, car from common.params import Params +from common.realtime import DT_CTRL from selfdrive.boardd.boardd import can_capnp_to_can_list, can_list_to_can_capnp from selfdrive.car.fingerprints import all_known_cars from selfdrive.car.car_helpers import interfaces @@ -27,8 +29,6 @@ PandaType = log.PandaState.PandaType NUM_JOBS = int(os.environ.get("NUM_JOBS", "1")) JOB_ID = int(os.environ.get("JOB_ID", "0")) -ROUTES = {rt.car_fingerprint: rt.route for rt in routes} - # TODO: get updated routes for these cars ignore_can_valid = [ HYUNDAI.SANTA_FE, @@ -39,25 +39,34 @@ ignore_addr_checks_valid = [ HYUNDAI.GENESIS_G70_2020, ] -@parameterized_class(('car_model'), [(car,) for i, car in enumerate(sorted(all_known_cars())) if i % NUM_JOBS == JOB_ID]) +# build list of test cases +routes_by_car = defaultdict(set) +for r in routes: + routes_by_car[r.car_fingerprint].add(r.route) + +test_cases: List[Tuple[str, Optional[str]]] = [] +for i, c in enumerate(sorted(all_known_cars())): + if i % NUM_JOBS == JOB_ID: + test_cases.extend((c, r) for r in routes_by_car.get(c, (None, ))) + + +@parameterized_class(('car_model', 'route'), test_cases) class TestCarModel(unittest.TestCase): @classmethod def setUpClass(cls): - if cls.car_model not in ROUTES: - # TODO: get routes for missing cars and remove this + if cls.route is None: if cls.car_model in non_tested_cars: print(f"Skipping tests for {cls.car_model}: missing route") raise unittest.SkipTest - else: - raise Exception(f"missing test route for car {cls.car_model}") + raise Exception(f"missing test route for {cls.car_model}") params = Params() params.clear_all() for seg in [2, 1, 0]: try: - lr = LogReader(get_url(ROUTES[cls.car_model], seg)) + lr = LogReader(get_url(cls.route, seg)) except Exception: continue @@ -73,16 +82,17 @@ class TestCarModel(unittest.TestCase): if msg.carParams.openpilotLongitudinalControl: params.put_bool("DisableRadar", True) - if len(can_msgs): + if len(can_msgs) > int(50 / DT_CTRL): break else: - raise Exception("Route not found or no CAN msgs found. Is it uploaded?") + raise Exception(f"Route {repr(cls.route)} not found or no CAN msgs found. Is it uploaded?") cls.can_msgs = sorted(can_msgs, key=lambda msg: msg.logMonoTime) cls.CarInterface, cls.CarController, cls.CarState = interfaces[cls.car_model] cls.CP = cls.CarInterface.get_params(cls.car_model, fingerprint, []) assert cls.CP + assert cls.CP.carFingerprint == cls.car_model def setUp(self): self.CI = self.CarInterface(self.CP, self.CarController, self.CarState) diff --git a/selfdrive/test/test_routes.py b/selfdrive/test/test_routes.py index 445005b88e..4040431920 100644 --- a/selfdrive/test/test_routes.py +++ b/selfdrive/test/test_routes.py @@ -2,7 +2,6 @@ from collections import namedtuple from selfdrive.car.chrysler.values import CAR as CHRYSLER -from selfdrive.car.ford.values import CAR as FORD from selfdrive.car.gm.values import CAR as GM from selfdrive.car.honda.values import CAR as HONDA from selfdrive.car.hyundai.values import CAR as HYUNDAI @@ -34,7 +33,7 @@ routes = [ TestRoute("8190c7275a24557b|2020-01-29--08-33-58", CHRYSLER.PACIFICA_2019_HYBRID), TestRoute("3d84727705fecd04|2021-05-25--08-38-56", CHRYSLER.PACIFICA_2020), - TestRoute("f1b4c567731f4a1b|2018-04-30--10-15-35", FORD.FUSION), + #TestRoute("f1b4c567731f4a1b|2018-04-30--10-15-35", FORD.FUSION), TestRoute("7cc2a8365b4dd8a9|2018-12-02--12-10-44", GM.ACADIA), TestRoute("aa20e335f61ba898|2019-02-05--16-59-04", GM.BUICK_REGAL), @@ -115,8 +114,6 @@ routes = [ TestRoute("5f5afb36036506e4|2019-05-14--02-09-54", TOYOTA.COROLLA_TSS2), TestRoute("5ceff72287a5c86c|2019-10-19--10-59-02", TOYOTA.COROLLAH_TSS2), TestRoute("d2525c22173da58b|2021-04-25--16-47-04", TOYOTA.PRIUS), - TestRoute("b0f5a01cf604185c|2017-12-18--20-32-32", TOYOTA.RAV4), - TestRoute("b0c9d2329ad1606b|2019-04-02--13-24-43", TOYOTA.RAV4), TestRoute("b14c5b4742e6fc85|2020-07-28--19-50-11", TOYOTA.RAV4), TestRoute("32a7df20486b0f70|2020-02-06--16-06-50", TOYOTA.RAV4H), TestRoute("cdf2f7de565d40ae|2019-04-25--03-53-41", TOYOTA.RAV4_TSS2), @@ -198,19 +195,3 @@ routes = [ TestRoute("6c14ee12b74823ce|2021-06-30--11-49-02", TESLA.AP1_MODELS), TestRoute("bb50caf5f0945ab1|2021-06-19--17-20-18", TESLA.AP2_MODELS), ] - -forced_dashcam_routes = [ - # Ford fusion - "f1b4c567731f4a1b|2018-04-18--11-29-37", - "f1b4c567731f4a1b|2018-04-30--10-15-35", - # Mazda CX5 - "32a319f057902bb3|2020-04-27--15-18-58", - # Mazda CX9 - "10b5a4b380434151|2020-08-26--17-11-45", - # Mazda3 - "74f1038827005090|2020-08-26--20-05-50", - # Mazda6 - "fb53c640f499b73d|2021-06-01--04-17-56", - # CX-9 2021 - "f6d5b1a9d7a1c92e|2021-07-08--06-56-59", -] diff --git a/tools/plotjuggler/layouts/controls_mismatch_debug.xml b/tools/plotjuggler/layouts/controls_mismatch_debug.xml new file mode 100644 index 0000000000..e50b5ffb34 --- /dev/null +++ b/tools/plotjuggler/layouts/controls_mismatch_debug.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +