From ac83318ac4f54c46deef02c69d46c549bd741cb3 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sat, 9 Dec 2023 14:28:32 -0800 Subject: [PATCH] test_models: randomize internal segment list (#30653) * randomize internal seg list segments * random * pytest-randomly sets random.seed to a consistent value for all the workers/processes * noeol * update * Revert "update" This reverts commit aff9a69c4e5e3934deebaa33986b42f44b55b002. * lock * don't randomize by default * remove random-order * strict * random * one fix * test * does nothing * rm tests * Revert "rm tests" This reverts commit b548e3fcd48e60538695506888d863c01b459d27. * (can't repro locally) just athena should be fine * bs1 * bs2 * bs3 * bs4 * bs5 * wrong way * no controls * no car * no board * controls? * crazy -common * Revert "crazy -common" This reverts commit 02365d712b3d09cab1893cce2261a4b418bb3851. * test athena * test athena 2 * test athena 3 * test athena 4 * test athena 5 * test athena 6 * test athena 7 * test athena 8 * test athena 9 * ?? * in one commit * common? * car and board * -controls -board * random-order * no board * revert * car/tests * least likely * try * try 2 * draft * draft * so much better * cmt * use randomly * not needed here * directly modify option works * bb * test time * Revert "test time" This reverts commit 2c5caabe2b470b47b7322e37800680b92773fccc. * tmut * i concur * revert --- conftest.py | 6 ++++++ poetry.lock | 18 ++++++++---------- pyproject.toml | 2 +- selfdrive/car/tests/test_models.py | 11 +++++------ 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/conftest.py b/conftest.py index 285d8b0238..3c566e3672 100644 --- a/conftest.py +++ b/conftest.py @@ -6,6 +6,12 @@ from openpilot.common.prefix import OpenpilotPrefix from openpilot.system.hardware import TICI +def pytest_sessionstart(session): + # TODO: fix tests and enable test order randomization + if session.config.pluginmanager.hasplugin('randomly'): + session.config.option.randomly_reorganize = False + + @pytest.fixture(scope="function", autouse=True) def openpilot_function_fixture(): starting_env = dict(os.environ) diff --git a/poetry.lock b/poetry.lock index 1461d5457c..1ec3847b18 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3309,8 +3309,6 @@ files = [ {file = "pygame-2.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e24d05184e4195fe5ebcdce8b18ecb086f00182b9ae460a86682d312ce8d31f"}, {file = "pygame-2.5.2-cp311-cp311-win32.whl", hash = "sha256:f02c1c7505af18d426d355ac9872bd5c916b27f7b0fe224749930662bea47a50"}, {file = "pygame-2.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:6d58c8cf937815d3b7cdc0fa9590c5129cb2c9658b72d00e8a4568dea2ff1d42"}, - {file = "pygame-2.5.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1a2a43802bb5e89ce2b3b775744e78db4f9a201bf8d059b946c61722840ceea8"}, - {file = "pygame-2.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1c289f2613c44fe70a1e40769de4a49c5ab5a29b9376f1692bb1a15c9c1c9bfa"}, {file = "pygame-2.5.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:074aa6c6e110c925f7f27f00c7733c6303407edc61d738882985091d1eb2ef17"}, {file = "pygame-2.5.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe0228501ec616779a0b9c4299e837877783e18df294dd690b9ab0eed3d8aaab"}, {file = "pygame-2.5.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31648d38ecdc2335ffc0e38fb18a84b3339730521505dac68514f83a1092e3f4"}, @@ -3719,18 +3717,18 @@ colorama = "*" pytest = ">=7.0" [[package]] -name = "pytest-random-order" -version = "1.1.0" -description = "Randomise the order in which pytest tests are run with some control over the randomness" +name = "pytest-randomly" +version = "3.15.0" +description = "Pytest plugin to randomly order tests and control random.seed." optional = false -python-versions = ">=3.5.0" +python-versions = ">=3.8" files = [ - {file = "pytest-random-order-1.1.0.tar.gz", hash = "sha256:dbe6debb9353a7af984cc9eddbeb3577dd4dbbcc1529a79e3d21f68ed9b45605"}, - {file = "pytest_random_order-1.1.0-py3-none-any.whl", hash = "sha256:6cb1e59ab0f798bb0c3488c11ae0c70d7d3340306a466d28b28ccd8ef8c20b7e"}, + {file = "pytest_randomly-3.15.0-py3-none-any.whl", hash = "sha256:0516f4344b29f4e9cdae8bce31c4aeebf59d0b9ef05927c33354ff3859eeeca6"}, + {file = "pytest_randomly-3.15.0.tar.gz", hash = "sha256:b908529648667ba5e54723088edd6f82252f540cc340d748d1fa985539687047"}, ] [package.dependencies] -pytest = ">=3.0.0" +pytest = "*" [[package]] name = "pytest-subtests" @@ -5011,4 +5009,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "~3.11" -content-hash = "f0abc0f53443c3b98eb06fc4dd0ff736e39197bc68c79db9aeffb4736657b676" +content-hash = "7f7244a89280a41937066ebf80aa64fcb375852484c9d034ea7f2de3eae4d90a" diff --git a/pyproject.toml b/pyproject.toml index 0116cb4c5e..c6a13c4b86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -151,7 +151,7 @@ pytest-subtests = "*" pytest-xdist = "*" pytest-timeout = "*" pytest-timeouts = "*" -pytest-random-order = "*" +pytest-randomly = "*" ruff = "*" scipy = "*" sphinx = "*" diff --git a/selfdrive/car/tests/test_models.py b/selfdrive/car/tests/test_models.py index 0ae104f9f5..aaaa6b9d79 100755 --- a/selfdrive/car/tests/test_models.py +++ b/selfdrive/car/tests/test_models.py @@ -3,6 +3,7 @@ import capnp import os import importlib import pytest +import random import unittest from collections import defaultdict, Counter from typing import List, Optional, Tuple @@ -49,12 +50,10 @@ def get_test_cases() -> List[Tuple[str, Optional[CarTestRoute]]]: with open(os.path.join(BASEDIR, INTERNAL_SEG_LIST), "r") as f: seg_list = f.read().splitlines() - cnt = INTERNAL_SEG_CNT or len(seg_list) - seg_list_iter = iter(seg_list[:cnt]) - - for platform in seg_list_iter: - platform = platform[2:] # get rid of comment - segment_name = SegmentName(next(seg_list_iter)) + seg_list_grouped = [(platform[2:], segment) for platform, segment in zip(seg_list[::2], seg_list[1::2], strict=True)] + seg_list_grouped = random.sample(seg_list_grouped, INTERNAL_SEG_CNT or len(seg_list_grouped)) + for platform, segment in seg_list_grouped: + segment_name = SegmentName(segment) test_cases.append((platform, CarTestRoute(segment_name.route_name.canonical_name, platform, segment=segment_name.segment_num))) return test_cases