diff --git a/selfdrive/car/tests/routes.py b/selfdrive/car/tests/routes.py index 27fdc9c705..d2ec6b68d7 100644 --- a/selfdrive/car/tests/routes.py +++ b/selfdrive/car/tests/routes.py @@ -26,7 +26,7 @@ non_tested_cars = [ HYUNDAI.KIA_OPTIMA_H, ] -TestRoute = namedtuple('TestRoute', ['route', 'car_fingerprint', 'segment'], defaults=(None,)) +TestRoute = namedtuple('TestRoute', ['route', 'car_model', 'segment'], defaults=(None,)) routes = [ TestRoute("efdf9af95e71cd84|2022-05-13--19-03-31", COMMA.BODY), diff --git a/selfdrive/car/tests/test_models.py b/selfdrive/car/tests/test_models.py index d2867b9f4f..9ab881279b 100755 --- a/selfdrive/car/tests/test_models.py +++ b/selfdrive/car/tests/test_models.py @@ -18,6 +18,7 @@ from selfdrive.car.hyundai.values import CAR as HYUNDAI from selfdrive.car.tests.routes import non_tested_cars, routes, TestRoute from selfdrive.test.openpilotci import get_url from tools.lib.logreader import LogReader +from tools.lib.route import Route from panda.tests.safety import libpandasafety_py from panda.tests.safety.common import package_can_msg @@ -35,7 +36,7 @@ ignore_addr_checks_valid = [ # build list of test cases routes_by_car = defaultdict(set) for r in routes: - routes_by_car[r.car_fingerprint].add(r) + routes_by_car[r.car_model].add(r) test_cases: List[Tuple[str, Optional[TestRoute]]] = [] for i, c in enumerate(sorted(all_known_cars())): @@ -45,12 +46,17 @@ for i, c in enumerate(sorted(all_known_cars())): SKIP_ENV_VAR = "SKIP_LONG_TESTS" -@parameterized_class(('car_model', 'test_route'), test_cases) -class TestCarModel(unittest.TestCase): +class TestCarModelBase(unittest.TestCase): + car_model = None + test_route = None + ci = True @unittest.skipIf(SKIP_ENV_VAR in os.environ, f"Long running test skipped. Unset {SKIP_ENV_VAR} to run") @classmethod def setUpClass(cls): + if cls.__name__ == 'TestCarModel' or cls.__name__.endswith('Base'): + raise unittest.SkipTest + if cls.test_route is None: if cls.car_model in non_tested_cars: print(f"Skipping tests for {cls.car_model}: missing route") @@ -64,7 +70,10 @@ class TestCarModel(unittest.TestCase): for seg in test_segs: try: - lr = LogReader(get_url(cls.test_route.route, seg)) + if cls.ci: + lr = LogReader(get_url(cls.test_route.route, seg)) + else: + lr = LogReader(Route(cls.test_route.route).log_paths()[seg]) except Exception: continue @@ -79,6 +88,8 @@ class TestCarModel(unittest.TestCase): elif msg.which() == "carParams": if msg.carParams.openpilotLongitudinalControl: disable_radar = True + if cls.car_model is None and not cls.ci: + cls.car_model = msg.carParams.carFingerprint if len(can_msgs) > int(50 / DT_CTRL): break @@ -249,5 +260,10 @@ class TestCarModel(unittest.TestCase): self.assertFalse(len(failed_checks), f"panda safety doesn't agree with openpilot: {failed_checks}") +@parameterized_class(('car_model', 'test_route'), test_cases) +class TestCarModel(TestCarModelBase): + pass + + if __name__ == "__main__": unittest.main() diff --git a/selfdrive/debug/test_car_model.py b/selfdrive/debug/test_car_model.py new file mode 100755 index 0000000000..9082dbe3d6 --- /dev/null +++ b/selfdrive/debug/test_car_model.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +import argparse +import sys +from typing import List, Tuple +import unittest + +from selfdrive.car.tests.routes import TestRoute +from selfdrive.car.tests.test_models import TestCarModel + + +def create_test_models_suite(routes: List[Tuple[str, TestRoute]], ci=False) -> unittest.TestSuite: + test_suite = unittest.TestSuite() + for car_model, test_route in routes: + # create new test case and discover tests + test_case_args = {"car_model": car_model, "test_route": test_route, "ci": ci} + CarModelTestCase = type("CarModelTestCase", (TestCarModel,), test_case_args) + test_suite.addTest(unittest.TestLoader().loadTestsFromTestCase(CarModelTestCase)) + return test_suite + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Test any route against common issues with a new car port. " + + "Uses selfdrive/car/tests/test_models.py") + parser.add_argument("route", help="Specify route to run tests on") + parser.add_argument("--car", help="Specify car model for test route") + parser.add_argument("--segment", type=int, nargs="?", help="Specify segment of route to test") + parser.add_argument("--ci", action="store_true", help="Attempt to get logs using openpilotci, need to specify car") + args = parser.parse_args() + if len(sys.argv) == 1: + parser.print_help() + sys.exit() + + test_route = TestRoute(args.route, args.car, segment=args.segment) + test_suite = create_test_models_suite([(args.car, test_route)], ci=args.ci) + + unittest.TextTestRunner().run(test_suite)