From 11492496fc9aa8af8fd42460815eff8adff52f0c Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 10 Aug 2023 00:45:23 -0700 Subject: [PATCH] test_models: run on route list (#26226) * move to openpilot * draft * support internal urls * not used * update seg list * regen segment list * regen seg list * regen * regen * regen * no dirty segments * regen seg list with no fuzzy or fixed fp * regen segments with hda2 exception * regen with added filters * regen without bad dongle * regen * regenerate! * regenerate, only OP enabled * regen * regenerate! * regenerate! * stash * use SegmentName * new route list * add temp comment * remove comment * let's see if this works * comment out other tests to speed up * fix agent * ? * hmm * oh whoops... * add matrix * fix * how many levels do we need? * fix values * matrix can't be in parallel * how about this * try this * Revert matrix Revert "try this" This reverts commit 8d0d91fd70c467c1fbb4e4e9aed511d15b2a30ef. Revert "how about this" This reverts commit a8e4fc53234824e77cbfb1e471277bc033e9dea1. Revert "matrix can't be in parallel" This reverts commit daaa6fcc3c75c74cbb90e97c565099a94123994c. Revert "fix values" This reverts commit df554b6a3371d124a574eb8d26bc51ef5b5b8fde. Revert "how many levels do we need?" This reverts commit 1a17320fa1c5e7220ef60e29981bbb3bb7da16c6. Revert "fix" This reverts commit e7eb6e404358fbd2eac3fea1901a8d30ea92d729. Revert "add matrix" This reverts commit a1b57e5725417d3c2f639f8edfc0c889b84b6753. * use pytest instead! (5 jobs is ~150 mins) * split lines, uncomment * This Sienna seg has a fault SDSU that stopped forwarding/sending msgs * picked a route with no PSCMStatus and no panda errors, recent routes are working * this cadillac was dashcammed (no radar) * opened an issue for this, it's 'expected' right now * small clean up * small clean up * i don't think that worked * is this needed? * add to new PC tests * cache * draft * Revert "draft" This reverts commit 3b7f740dd4883118747300bc3687074c2d3c2116. * probably should be function * draft * clean up * add todo * 600 random segments * debug * does this fix pythonpath issues? fix * try this * mount? * pytest again! * no need for PYTHONPATH now * Update Jenkinsfile * ? * convention * clean up * would be even more complex (have to unset ci which is class level) * track * is lfs pulled at all? * ah no it's not old-commit-hash: 70b1e9dd756cd2f6929cdf33dd1061a33f3b52b1 --- .gitattributes | 1 + Jenkinsfile | 6 ++-- selfdrive/car/tests/test_models.py | 42 +++++++++++++++++------- selfdrive/car/tests/test_models_segs.txt | 3 ++ 4 files changed, 39 insertions(+), 13 deletions(-) create mode 100644 selfdrive/car/tests/test_models_segs.txt diff --git a/.gitattributes b/.gitattributes index aac566e6d9..76bb2d2aee 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,5 +1,6 @@ *.dlc filter=lfs diff=lfs merge=lfs -text *.onnx filter=lfs diff=lfs merge=lfs -text +selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text *.ico filter=lfs diff=lfs merge=lfs -text *.svg filter=lfs diff=lfs merge=lfs -text *.png filter=lfs diff=lfs merge=lfs -text diff --git a/Jenkinsfile b/Jenkinsfile index 3eab4e35e2..6a834288c2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -143,18 +143,20 @@ pipeline { } */ - stage('scons build test') { + stage('PC tests') { agent { dockerfile { filename 'Dockerfile.openpilot_base' - args '--user=root' + args '--user=root -v /tmp/comma_download_cache:/tmp/comma_download_cache' } } steps { sh "git config --global --add safe.directory '*'" sh "git submodule update --init --depth=1 --recursive" + sh "git lfs pull" sh "scons --clean && scons --no-cache -j42" sh "scons --clean && scons --no-cache --random -j42" + sh "INTERNAL_SEG_LIST=selfdrive/car/tests/test_models_segs.txt FILEREADER_CACHE=1 pytest -n42 --dist=loadscope selfdrive/car/tests/test_models.py" } post { diff --git a/selfdrive/car/tests/test_models.py b/selfdrive/car/tests/test_models.py index cf7727dafe..a2a8ed5eb7 100755 --- a/selfdrive/car/tests/test_models.py +++ b/selfdrive/car/tests/test_models.py @@ -8,6 +8,7 @@ from typing import List, Optional, Tuple from parameterized import parameterized_class from cereal import log, car +from common.basedir import BASEDIR from common.realtime import DT_CTRL from selfdrive.car.fingerprints import all_known_cars from selfdrive.car.car_helpers import interfaces @@ -17,7 +18,7 @@ from selfdrive.car.hyundai.values import CAR as HYUNDAI from selfdrive.car.tests.routes import non_tested_cars, routes, CarTestRoute from selfdrive.test.openpilotci import get_url from tools.lib.logreader import LogReader -from tools.lib.route import Route +from tools.lib.route import Route, SegmentName, RouteName from panda.tests.libpanda import libpanda_py @@ -25,21 +26,37 @@ PandaType = log.PandaState.PandaType NUM_JOBS = int(os.environ.get("NUM_JOBS", "1")) JOB_ID = int(os.environ.get("JOB_ID", "0")) +INTERNAL_SEG_LIST = os.environ.get("INTERNAL_SEG_LIST", "") ignore_addr_checks_valid = [ GM.BUICK_REGAL, HYUNDAI.GENESIS_G70_2020, ] -# build list of test cases -routes_by_car = defaultdict(set) -for r in routes: - routes_by_car[r.car_model].add(r) -test_cases: List[Tuple[str, Optional[CarTestRoute]]] = [] -for i, c in enumerate(sorted(all_known_cars())): - if i % NUM_JOBS == JOB_ID: - test_cases.extend(sorted((c, r) for r in routes_by_car.get(c, (None, )))) +def get_test_cases(): + # build list of test cases + test_cases: List[Tuple[str, Optional[CarTestRoute]]] = [] + if not len(INTERNAL_SEG_LIST): + routes_by_car = defaultdict(set) + for r in routes: + routes_by_car[r.car_model].add(r) + + for i, c in enumerate(sorted(all_known_cars())): + if i % NUM_JOBS == JOB_ID: + test_cases.extend(sorted((c, r) for r in routes_by_car.get(c, (None,)))) + + else: + with open(os.path.join(BASEDIR, INTERNAL_SEG_LIST), "r") as f: + seg_list = iter(f.read().splitlines()) + + for platform in seg_list: + platform = platform[2:] # get rid of comment + segment_name = SegmentName(next(seg_list)) + test_cases.append((platform, CarTestRoute(segment_name.route_name.canonical_name, platform, + segment=segment_name.segment_num))) + return test_cases + SKIP_ENV_VAR = "SKIP_LONG_TESTS" @@ -72,7 +89,10 @@ class TestCarModelBase(unittest.TestCase): for seg in test_segs: try: - if cls.ci: + if len(INTERNAL_SEG_LIST): + route_name = RouteName(cls.test_route.route) + lr = LogReader(f"cd:/{route_name.dongle_id}/{route_name.time_str}/{seg}/rlog.bz2") + elif cls.ci: lr = LogReader(get_url(cls.test_route.route, seg)) else: lr = LogReader(Route(cls.test_route.route).log_paths()[seg]) @@ -314,7 +334,7 @@ class TestCarModelBase(unittest.TestCase): self.assertFalse(len(failed_checks), f"panda safety doesn't agree with openpilot: {failed_checks}") -@parameterized_class(('car_model', 'test_route'), test_cases) +@parameterized_class(('car_model', 'test_route'), get_test_cases()) class TestCarModel(TestCarModelBase): pass diff --git a/selfdrive/car/tests/test_models_segs.txt b/selfdrive/car/tests/test_models_segs.txt new file mode 100644 index 0000000000..68841af007 --- /dev/null +++ b/selfdrive/car/tests/test_models_segs.txt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:636df10e41e8c6668678a3aa7098afad8c160ea7b75c5a0473f6709db1c702dc +size 39373