test_models: run on route list (#26226)

* move to openpilot

* draft

* support internal urls

* not used

* update seg list

* regen segment list

* regen seg list

* regen

* regen

* regen

* no dirty segments

* regen seg list with no fuzzy or fixed fp

* regen segments with hda2 exception

* regen with added filters

* regen without bad dongle

* regen

* regenerate!

* regenerate, only OP enabled

* regen

* regenerate!

* regenerate!

* stash

* use SegmentName

* new route list

* add temp comment

* remove comment

* let's see if this works

* comment out other tests to speed up

* fix agent

* ?

* hmm

* oh whoops...

* add matrix

* fix

* how many levels do we need?

* fix values

* matrix can't be in parallel

* how about this

* try this

* Revert matrix

Revert "try this"

This reverts commit 8d0d91fd70c467c1fbb4e4e9aed511d15b2a30ef.

Revert "how about this"

This reverts commit a8e4fc53234824e77cbfb1e471277bc033e9dea1.

Revert "matrix can't be in parallel"

This reverts commit daaa6fcc3c75c74cbb90e97c565099a94123994c.

Revert "fix values"

This reverts commit df554b6a3371d124a574eb8d26bc51ef5b5b8fde.

Revert "how many levels do we need?"

This reverts commit 1a17320fa1c5e7220ef60e29981bbb3bb7da16c6.

Revert "fix"

This reverts commit e7eb6e404358fbd2eac3fea1901a8d30ea92d729.

Revert "add matrix"

This reverts commit a1b57e5725417d3c2f639f8edfc0c889b84b6753.

* use pytest instead! (5 jobs is ~150 mins)

* split lines, uncomment

* This Sienna seg has a fault SDSU that stopped forwarding/sending msgs

* picked a route with no PSCMStatus and no panda errors, recent routes are working

* this cadillac was dashcammed (no radar)

* opened an issue for this, it's 'expected' right now

* small clean up

* small clean up

* i don't think that worked

* is this needed?

* add to new PC tests

* cache

* draft

* Revert "draft"

This reverts commit 3b7f740dd4883118747300bc3687074c2d3c2116.

* probably should be function

* draft

* clean up

* add todo

* 600 random segments

* debug

* does this fix pythonpath issues?

fix

* try this

* mount?

* pytest again!

* no need for PYTHONPATH now

* Update Jenkinsfile

* ?

* convention

* clean up

* would be even more complex (have to unset ci which is class level)

* track

* is lfs pulled at all?

* ah no it's not
old-commit-hash: 70b1e9dd75

.gitattributes

@@ -1,5 +1,6 @@
 *.dlc filter=lfs diff=lfs merge=lfs -text
 *.onnx filter=lfs diff=lfs merge=lfs -text
+selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
 *.ico filter=lfs diff=lfs merge=lfs -text
 *.svg filter=lfs diff=lfs merge=lfs -text
 *.png filter=lfs diff=lfs merge=lfs -text

Jenkinsfile

@@ -143,18 +143,20 @@ pipeline {
         }
         */
 
-        stage('scons build test') {
+        stage('PC tests') {
           agent {
             dockerfile {
               filename 'Dockerfile.openpilot_base'
-              args '--user=root'
+              args '--user=root -v /tmp/comma_download_cache:/tmp/comma_download_cache'
             }
           }
           steps {
             sh "git config --global --add safe.directory '*'"
             sh "git submodule update --init --depth=1 --recursive"
+            sh "git lfs pull"
             sh "scons --clean && scons --no-cache -j42"
             sh "scons --clean && scons --no-cache --random -j42"
+            sh "INTERNAL_SEG_LIST=selfdrive/car/tests/test_models_segs.txt FILEREADER_CACHE=1 pytest -n42 --dist=loadscope selfdrive/car/tests/test_models.py"
           }
 
           post {

selfdrive/car/tests/test_models.py

@@ -8,6 +8,7 @@ from typing import List, Optional, Tuple
 from parameterized import parameterized_class
 
 from cereal import log, car
+from common.basedir import BASEDIR
 from common.realtime import DT_CTRL
 from selfdrive.car.fingerprints import all_known_cars
 from selfdrive.car.car_helpers import interfaces
@@ -17,7 +18,7 @@ from selfdrive.car.hyundai.values import CAR as HYUNDAI
 from selfdrive.car.tests.routes import non_tested_cars, routes, CarTestRoute
 from selfdrive.test.openpilotci import get_url
 from tools.lib.logreader import LogReader
-from tools.lib.route import Route
+from tools.lib.route import Route, SegmentName, RouteName
 
 from panda.tests.libpanda import libpanda_py
 
@@ -25,21 +26,37 @@ PandaType = log.PandaState.PandaType
 
 NUM_JOBS = int(os.environ.get("NUM_JOBS", "1"))
 JOB_ID = int(os.environ.get("JOB_ID", "0"))
+INTERNAL_SEG_LIST = os.environ.get("INTERNAL_SEG_LIST", "")
 
 ignore_addr_checks_valid = [
   GM.BUICK_REGAL,
   HYUNDAI.GENESIS_G70_2020,
 ]
 
-# build list of test cases
+
-routes_by_car = defaultdict(set)
+def get_test_cases():
-for r in routes:
+  # build list of test cases
+  test_cases: List[Tuple[str, Optional[CarTestRoute]]] = []
+  if not len(INTERNAL_SEG_LIST):
+    routes_by_car = defaultdict(set)
+    for r in routes:
       routes_by_car[r.car_model].add(r)
 
-test_cases: List[Tuple[str, Optional[CarTestRoute]]] = []
+    for i, c in enumerate(sorted(all_known_cars())):
-for i, c in enumerate(sorted(all_known_cars())):
       if i % NUM_JOBS == JOB_ID:
-    test_cases.extend(sorted((c, r) for r in routes_by_car.get(c, (None, ))))
+        test_cases.extend(sorted((c, r) for r in routes_by_car.get(c, (None,))))
+
+  else:
+    with open(os.path.join(BASEDIR, INTERNAL_SEG_LIST), "r") as f:
+      seg_list = iter(f.read().splitlines())
+
+    for platform in seg_list:
+      platform = platform[2:]  # get rid of comment
+      segment_name = SegmentName(next(seg_list))
+      test_cases.append((platform, CarTestRoute(segment_name.route_name.canonical_name, platform,
+                                                segment=segment_name.segment_num)))
+  return test_cases
+
 
 SKIP_ENV_VAR = "SKIP_LONG_TESTS"
 
@@ -72,7 +89,10 @@ class TestCarModelBase(unittest.TestCase):
 
     for seg in test_segs:
       try:
-        if cls.ci:
+        if len(INTERNAL_SEG_LIST):
+          route_name = RouteName(cls.test_route.route)
+          lr = LogReader(f"cd:/{route_name.dongle_id}/{route_name.time_str}/{seg}/rlog.bz2")
+        elif cls.ci:
           lr = LogReader(get_url(cls.test_route.route, seg))
         else:
           lr = LogReader(Route(cls.test_route.route).log_paths()[seg])
@@ -314,7 +334,7 @@ class TestCarModelBase(unittest.TestCase):
     self.assertFalse(len(failed_checks), f"panda safety doesn't agree with openpilot: {failed_checks}")
 
 
-@parameterized_class(('car_model', 'test_route'), test_cases)
+@parameterized_class(('car_model', 'test_route'), get_test_cases())
 class TestCarModel(TestCarModelBase):
   pass
 

selfdrive/car/tests/test_models_segs.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:636df10e41e8c6668678a3aa7098afad8c160ea7b75c5a0473f6709db1c702dc
+size 39373