test_models: support for running on any route (#24002)

* run test_models on route

* clean up a bit

clean up a bit

clean up a bit

* remove

remove

* make route positional

* fixes from merge

* not working as I expected

* finally working, easiest way seems to make a new subclass dynamically

* bring back routes

* remove comments

* revert skiptest

* fix subclass name

* car first

* this should be here...

* this should work

* comment

* pytest doesn't support dynamically loading with load_tests

* minimize test_models.py diff, and make new file for running on a route

* fix static analysis

* remove print

* clean up a tiny bit

* rename and make required

* auto detect car fingerprint if not given

* move to location that makes more sense

* Add ci argument for running route from routes.py

selfdrive/car/tests/routes.py

@@ -26,7 +26,7 @@ non_tested_cars = [
   HYUNDAI.KIA_OPTIMA_H,
 ]
 
-TestRoute = namedtuple('TestRoute', ['route', 'car_fingerprint', 'segment'], defaults=(None,))
+TestRoute = namedtuple('TestRoute', ['route', 'car_model', 'segment'], defaults=(None,))
 
 routes = [
   TestRoute("efdf9af95e71cd84|2022-05-13--19-03-31", COMMA.BODY),

selfdrive/car/tests/test_models.py

@@ -18,6 +18,7 @@ from selfdrive.car.hyundai.values import CAR as HYUNDAI
 from selfdrive.car.tests.routes import non_tested_cars, routes, TestRoute
 from selfdrive.test.openpilotci import get_url
 from tools.lib.logreader import LogReader
+from tools.lib.route import Route
 
 from panda.tests.safety import libpandasafety_py
 from panda.tests.safety.common import package_can_msg
@@ -35,7 +36,7 @@ ignore_addr_checks_valid = [
 # build list of test cases
 routes_by_car = defaultdict(set)
 for r in routes:
-  routes_by_car[r.car_fingerprint].add(r)
+  routes_by_car[r.car_model].add(r)
 
 test_cases: List[Tuple[str, Optional[TestRoute]]] = []
 for i, c in enumerate(sorted(all_known_cars())):
@@ -45,12 +46,17 @@ for i, c in enumerate(sorted(all_known_cars())):
 SKIP_ENV_VAR = "SKIP_LONG_TESTS"
 
 
-@parameterized_class(('car_model', 'test_route'), test_cases)
+class TestCarModelBase(unittest.TestCase):
-class TestCarModel(unittest.TestCase):
+  car_model = None
+  test_route = None
+  ci = True
 
   @unittest.skipIf(SKIP_ENV_VAR in os.environ, f"Long running test skipped. Unset {SKIP_ENV_VAR} to run")
   @classmethod
   def setUpClass(cls):
+    if cls.__name__ == 'TestCarModel' or cls.__name__.endswith('Base'):
+      raise unittest.SkipTest
+
     if cls.test_route is None:
       if cls.car_model in non_tested_cars:
         print(f"Skipping tests for {cls.car_model}: missing route")
@@ -64,7 +70,10 @@ class TestCarModel(unittest.TestCase):
 
     for seg in test_segs:
       try:
-        lr = LogReader(get_url(cls.test_route.route, seg))
+        if cls.ci:
+          lr = LogReader(get_url(cls.test_route.route, seg))
+        else:
+          lr = LogReader(Route(cls.test_route.route).log_paths()[seg])
       except Exception:
         continue
 
@@ -79,6 +88,8 @@ class TestCarModel(unittest.TestCase):
         elif msg.which() == "carParams":
           if msg.carParams.openpilotLongitudinalControl:
             disable_radar = True
+          if cls.car_model is None and not cls.ci:
+            cls.car_model = msg.carParams.carFingerprint
 
       if len(can_msgs) > int(50 / DT_CTRL):
         break
@@ -249,5 +260,10 @@ class TestCarModel(unittest.TestCase):
     self.assertFalse(len(failed_checks), f"panda safety doesn't agree with openpilot: {failed_checks}")
 
 
+@parameterized_class(('car_model', 'test_route'), test_cases)
+class TestCarModel(TestCarModelBase):
+  pass
+
+
 if __name__ == "__main__":
   unittest.main()

selfdrive/debug/test_car_model.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+import argparse
+import sys
+from typing import List, Tuple
+import unittest
+
+from selfdrive.car.tests.routes import TestRoute
+from selfdrive.car.tests.test_models import TestCarModel
+
+
+def create_test_models_suite(routes: List[Tuple[str, TestRoute]], ci=False) -> unittest.TestSuite:
+  test_suite = unittest.TestSuite()
+  for car_model, test_route in routes:
+    # create new test case and discover tests
+    test_case_args = {"car_model": car_model, "test_route": test_route, "ci": ci}
+    CarModelTestCase = type("CarModelTestCase", (TestCarModel,), test_case_args)
+    test_suite.addTest(unittest.TestLoader().loadTestsFromTestCase(CarModelTestCase))
+  return test_suite
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="Test any route against common issues with a new car port. " +
+                                               "Uses selfdrive/car/tests/test_models.py")
+  parser.add_argument("route", help="Specify route to run tests on")
+  parser.add_argument("--car", help="Specify car model for test route")
+  parser.add_argument("--segment", type=int, nargs="?", help="Specify segment of route to test")
+  parser.add_argument("--ci", action="store_true", help="Attempt to get logs using openpilotci, need to specify car")
+  args = parser.parse_args()
+  if len(sys.argv) == 1:
+    parser.print_help()
+    sys.exit()
+
+  test_route = TestRoute(args.route, args.car, segment=args.segment)
+  test_suite = create_test_models_suite([(args.car, test_route)], ci=args.ci)
+
+  unittest.TextTestRunner().run(test_suite)