test_models: run on route list (#26226)

* move to openpilot

* draft

* support internal urls

* not used

* update seg list

* regen segment list

* regen seg list

* regen

* regen

* regen

* no dirty segments

* regen seg list with no fuzzy or fixed fp

* regen segments with hda2 exception

* regen with added filters

* regen without bad dongle

* regen

* regenerate!

* regenerate, only OP enabled

* regen

* regenerate!

* regenerate!

* stash

* use SegmentName

* new route list

* add temp comment

* remove comment

* let's see if this works

* comment out other tests to speed up

* fix agent

* ?

* hmm

* oh whoops...

* add matrix

* fix

* how many levels do we need?

* fix values

* matrix can't be in parallel

* how about this

* try this

* Revert matrix

Revert "try this"

This reverts commit 8d0d91fd70.

Revert "how about this"

This reverts commit a8e4fc5323.

Revert "matrix can't be in parallel"

This reverts commit daaa6fcc3c.

Revert "fix values"

This reverts commit df554b6a33.

Revert "how many levels do we need?"

This reverts commit 1a17320fa1.

Revert "fix"

This reverts commit e7eb6e4043.

Revert "add matrix"

This reverts commit a1b57e5725.

* use pytest instead! (5 jobs is ~150 mins)

* split lines, uncomment

* This Sienna seg has a fault SDSU that stopped forwarding/sending msgs

* picked a route with no PSCMStatus and no panda errors, recent routes are working

* this cadillac was dashcammed (no radar)

* opened an issue for this, it's 'expected' right now

* small clean up

* small clean up

* i don't think that worked

* is this needed?

* add to new PC tests

* cache

* draft

* Revert "draft"

This reverts commit 3b7f740dd4.

* probably should be function

* draft

* clean up

* add todo

* 600 random segments

* debug

* does this fix pythonpath issues?

fix

* try this

* mount?

* pytest again!

* no need for PYTHONPATH now

* Update Jenkinsfile

* ?

* convention

* clean up

* would be even more complex (have to unset ci which is class level)

* track

* is lfs pulled at all?

* ah no it's not

.gitattributes

@@ -1,2 +1,3 @@
 *.dlc filter=lfs diff=lfs merge=lfs -text
 *.onnx filter=lfs diff=lfs merge=lfs -text
+selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text

Jenkinsfile

@@ -143,18 +143,20 @@ pipeline {
         }
         */
 
-        stage('scons build test') {
+        stage('PC tests') {
           agent {
             dockerfile {
               filename 'Dockerfile.openpilot_base'
-              args '--user=root'
+              args '--user=root -v /tmp/comma_download_cache:/tmp/comma_download_cache'
             }
           }
           steps {
             sh "git config --global --add safe.directory '*'"
             sh "git submodule update --init --depth=1 --recursive"
+            sh "git lfs pull"
             sh "scons --clean && scons --no-cache -j42"
             sh "scons --clean && scons --no-cache --random -j42"
+            sh "INTERNAL_SEG_LIST=selfdrive/car/tests/test_models_segs.txt FILEREADER_CACHE=1 pytest -n42 --dist=loadscope selfdrive/car/tests/test_models.py"
           }
 
           post {

selfdrive/car/tests/test_models.py

@@ -8,6 +8,7 @@ from typing import List, Optional, Tuple
 from parameterized import parameterized_class
 
 from cereal import log, car
+from common.basedir import BASEDIR
 from common.realtime import DT_CTRL
 from selfdrive.car.fingerprints import all_known_cars
 from selfdrive.car.car_helpers import interfaces
@@ -17,7 +18,7 @@ from selfdrive.car.hyundai.values import CAR as HYUNDAI
 from selfdrive.car.tests.routes import non_tested_cars, routes, CarTestRoute
 from selfdrive.test.openpilotci import get_url
 from tools.lib.logreader import LogReader
-from tools.lib.route import Route
+from tools.lib.route import Route, SegmentName, RouteName
 
 from panda.tests.libpanda import libpanda_py
 
@@ -25,22 +26,38 @@ PandaType = log.PandaState.PandaType
 
 NUM_JOBS = int(os.environ.get("NUM_JOBS", "1"))
 JOB_ID = int(os.environ.get("JOB_ID", "0"))
+INTERNAL_SEG_LIST = os.environ.get("INTERNAL_SEG_LIST", "")
 
 ignore_addr_checks_valid = [
   GM.BUICK_REGAL,
   HYUNDAI.GENESIS_G70_2020,
 ]
 
+
+def get_test_cases():
   # build list of test cases
+  test_cases: List[Tuple[str, Optional[CarTestRoute]]] = []
+  if not len(INTERNAL_SEG_LIST):
     routes_by_car = defaultdict(set)
     for r in routes:
       routes_by_car[r.car_model].add(r)
 
-test_cases: List[Tuple[str, Optional[CarTestRoute]]] = []
     for i, c in enumerate(sorted(all_known_cars())):
       if i % NUM_JOBS == JOB_ID:
         test_cases.extend(sorted((c, r) for r in routes_by_car.get(c, (None,))))
 
+  else:
+    with open(os.path.join(BASEDIR, INTERNAL_SEG_LIST), "r") as f:
+      seg_list = iter(f.read().splitlines())
+
+    for platform in seg_list:
+      platform = platform[2:]  # get rid of comment
+      segment_name = SegmentName(next(seg_list))
+      test_cases.append((platform, CarTestRoute(segment_name.route_name.canonical_name, platform,
+                                                segment=segment_name.segment_num)))
+  return test_cases
+
+
 SKIP_ENV_VAR = "SKIP_LONG_TESTS"
 
 
@@ -72,7 +89,10 @@ class TestCarModelBase(unittest.TestCase):
 
     for seg in test_segs:
       try:
-        if cls.ci:
+        if len(INTERNAL_SEG_LIST):
+          route_name = RouteName(cls.test_route.route)
+          lr = LogReader(f"cd:/{route_name.dongle_id}/{route_name.time_str}/{seg}/rlog.bz2")
+        elif cls.ci:
           lr = LogReader(get_url(cls.test_route.route, seg))
         else:
           lr = LogReader(Route(cls.test_route.route).log_paths()[seg])
@@ -314,7 +334,7 @@ class TestCarModelBase(unittest.TestCase):
     self.assertFalse(len(failed_checks), f"panda safety doesn't agree with openpilot: {failed_checks}")
 
 
-@parameterized_class(('car_model', 'test_route'), test_cases)
+@parameterized_class(('car_model', 'test_route'), get_test_cases())
 class TestCarModel(TestCarModelBase):
   pass
 

selfdrive/car/tests/test_models_segs.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:636df10e41e8c6668678a3aa7098afad8c160ea7b75c5a0473f6709db1c702dc
+size 39373