test car models improvements (#23668)

* run on all routes

* min segment length

* invalid

* revert that

selfdrive/test/test_models.py

@@ -4,10 +4,12 @@ import os
 import importlib
 import unittest
 from collections import defaultdict, Counter
+from typing import List, Optional, Tuple
 from parameterized import parameterized_class
 
 from cereal import log, car
 from common.params import Params
+from common.realtime import DT_CTRL
 from selfdrive.boardd.boardd import can_capnp_to_can_list, can_list_to_can_capnp
 from selfdrive.car.fingerprints import all_known_cars
 from selfdrive.car.car_helpers import interfaces
@@ -27,8 +29,6 @@ PandaType = log.PandaState.PandaType
 NUM_JOBS = int(os.environ.get("NUM_JOBS", "1"))
 JOB_ID = int(os.environ.get("JOB_ID", "0"))
 
-ROUTES = {rt.car_fingerprint: rt.route for rt in routes}
-
 # TODO: get updated routes for these cars
 ignore_can_valid = [
   HYUNDAI.SANTA_FE,
@@ -39,25 +39,34 @@ ignore_addr_checks_valid = [
   HYUNDAI.GENESIS_G70_2020,
 ]
 
-@parameterized_class(('car_model'), [(car,) for i, car in enumerate(sorted(all_known_cars())) if i % NUM_JOBS == JOB_ID])
+# build list of test cases
+routes_by_car = defaultdict(set)
+for r in routes:
+  routes_by_car[r.car_fingerprint].add(r.route)
+
+test_cases: List[Tuple[str, Optional[str]]] = []
+for i, c in enumerate(sorted(all_known_cars())):
+  if i % NUM_JOBS == JOB_ID:
+    test_cases.extend((c, r) for r in routes_by_car.get(c, (None, )))
+
+
+@parameterized_class(('car_model', 'route'), test_cases)
 class TestCarModel(unittest.TestCase):
 
   @classmethod
   def setUpClass(cls):
-    if cls.car_model not in ROUTES:
-      # TODO: get routes for missing cars and remove this
+    if cls.route is None:
       if cls.car_model in non_tested_cars:
         print(f"Skipping tests for {cls.car_model}: missing route")
         raise unittest.SkipTest
-      else:
-        raise Exception(f"missing test route for car {cls.car_model}")
+      raise Exception(f"missing test route for {cls.car_model}")
 
     params = Params()
     params.clear_all()
 
     for seg in [2, 1, 0]:
       try:
-        lr = LogReader(get_url(ROUTES[cls.car_model], seg))
+        lr = LogReader(get_url(cls.route, seg))
       except Exception:
         continue
 
@@ -73,16 +82,17 @@ class TestCarModel(unittest.TestCase):
           if msg.carParams.openpilotLongitudinalControl:
             params.put_bool("DisableRadar", True)
 
-      if len(can_msgs):
+      if len(can_msgs) > int(50 / DT_CTRL):
         break
     else:
-      raise Exception("Route not found or no CAN msgs found. Is it uploaded?")
+      raise Exception(f"Route {repr(cls.route)} not found or no CAN msgs found. Is it uploaded?")
 
     cls.can_msgs = sorted(can_msgs, key=lambda msg: msg.logMonoTime)
 
     cls.CarInterface, cls.CarController, cls.CarState = interfaces[cls.car_model]
     cls.CP = cls.CarInterface.get_params(cls.car_model, fingerprint, [])
     assert cls.CP
+    assert cls.CP.carFingerprint == cls.car_model
 
   def setUp(self):
     self.CI = self.CarInterface(self.CP, self.CarController, self.CarState)

selfdrive/test/test_routes.py

@@ -2,7 +2,6 @@
 from collections import namedtuple
 
 from selfdrive.car.chrysler.values import CAR as CHRYSLER
-from selfdrive.car.ford.values import CAR as FORD
 from selfdrive.car.gm.values import CAR as GM
 from selfdrive.car.honda.values import CAR as HONDA
 from selfdrive.car.hyundai.values import CAR as HYUNDAI
@@ -34,7 +33,7 @@ routes = [
   TestRoute("8190c7275a24557b|2020-01-29--08-33-58", CHRYSLER.PACIFICA_2019_HYBRID),
   TestRoute("3d84727705fecd04|2021-05-25--08-38-56", CHRYSLER.PACIFICA_2020),
 
-  TestRoute("f1b4c567731f4a1b|2018-04-30--10-15-35", FORD.FUSION),
+  #TestRoute("f1b4c567731f4a1b|2018-04-30--10-15-35", FORD.FUSION),
 
   TestRoute("7cc2a8365b4dd8a9|2018-12-02--12-10-44", GM.ACADIA),
   TestRoute("aa20e335f61ba898|2019-02-05--16-59-04", GM.BUICK_REGAL),
@@ -115,8 +114,6 @@ routes = [
   TestRoute("5f5afb36036506e4|2019-05-14--02-09-54", TOYOTA.COROLLA_TSS2),
   TestRoute("5ceff72287a5c86c|2019-10-19--10-59-02", TOYOTA.COROLLAH_TSS2),
   TestRoute("d2525c22173da58b|2021-04-25--16-47-04", TOYOTA.PRIUS),
-  TestRoute("b0f5a01cf604185c|2017-12-18--20-32-32", TOYOTA.RAV4),
-  TestRoute("b0c9d2329ad1606b|2019-04-02--13-24-43", TOYOTA.RAV4),
   TestRoute("b14c5b4742e6fc85|2020-07-28--19-50-11", TOYOTA.RAV4),
   TestRoute("32a7df20486b0f70|2020-02-06--16-06-50", TOYOTA.RAV4H),
   TestRoute("cdf2f7de565d40ae|2019-04-25--03-53-41", TOYOTA.RAV4_TSS2),
@@ -198,19 +195,3 @@ routes = [
   TestRoute("6c14ee12b74823ce|2021-06-30--11-49-02", TESLA.AP1_MODELS),
   TestRoute("bb50caf5f0945ab1|2021-06-19--17-20-18", TESLA.AP2_MODELS),
 ]
-
-forced_dashcam_routes = [
-  # Ford fusion
-  "f1b4c567731f4a1b|2018-04-18--11-29-37",
-  "f1b4c567731f4a1b|2018-04-30--10-15-35",
-  # Mazda CX5
-  "32a319f057902bb3|2020-04-27--15-18-58",
-  # Mazda CX9
-  "10b5a4b380434151|2020-08-26--17-11-45",
-  # Mazda3
-  "74f1038827005090|2020-08-26--20-05-50",
-  # Mazda6
-  "fb53c640f499b73d|2021-06-01--04-17-56",
-  # CX-9 2021
-  "f6d5b1a9d7a1c92e|2021-07-08--06-56-59",
-]

tools/plotjuggler/layouts/controls_mismatch_debug.xml

@@ -0,0 +1,46 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<root>
+ <tabbed_widget name="Main Window" parent="main_window">
+  <Tab containers="1" tab_name="tab1">
+   <Container>
+    <DockSplitter sizes="0.332965;0.334071;0.332965" orientation="-" count="3">
+     <DockArea name="...">
+      <plot style="Lines" mode="TimeSeries">
+       <range left="0.000000" top="1.025000" bottom="-0.025000" right="59.900659"/>
+       <limitY/>
+       <curve name="/controlsState/enabled" color="#1f77b4"/>
+       <curve name="/pandaStates/0/controlsAllowed" color="#d62728"/>
+      </plot>
+     </DockArea>
+     <DockArea name="...">
+      <plot style="Lines" mode="TimeSeries">
+       <range left="0.000000" top="-10.627637" bottom="-15.150799" right="59.900659"/>
+       <limitY/>
+       <curve name="/controlsState/cumLagMs" color="#9467bd"/>
+      </plot>
+     </DockArea>
+     <DockArea name="...">
+      <plot style="Lines" mode="TimeSeries">
+       <range left="0.000447" top="1.025000" bottom="-0.025000" right="59.900939"/>
+       <limitY/>
+       <curve name="/carState/gasPressed" color="#1ac938"/>
+       <curve name="/carState/brakePressed" color="#ff7f0e"/>
+      </plot>
+     </DockArea>
+    </DockSplitter>
+   </Container>
+  </Tab>
+  <currentTabIndex index="0"/>
+ </tabbed_widget>
+ <use_relative_time_offset enabled="1"/>
+ <!-- - - - - - - - - - - - - - - -->
+ <!-- - - - - - - - - - - - - - - -->
+ <Plugins>
+  <plugin ID="DataLoad Rlog"/>
+  <plugin ID="Cereal Subscriber"/>
+ </Plugins>
+ <!-- - - - - - - - - - - - - - - -->
+ <!-- - - - - - - - - - - - - - - -->
+ <!-- - - - - - - - - - - - - - - -->
+</root>
+