From e46c20ac61684e5d6796471119f27a79f30be9a4 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sat, 9 Dec 2023 14:28:32 -0800 Subject: [PATCH] test_models: randomize internal segment list (#30653) * randomize internal seg list segments * random * pytest-randomly sets random.seed to a consistent value for all the workers/processes * noeol * update * Revert "update" This reverts commit aff9a69c4e5e3934deebaa33986b42f44b55b002. * lock * don't randomize by default * remove random-order * strict * random * one fix * test * does nothing * rm tests * Revert "rm tests" This reverts commit b548e3fcd48e60538695506888d863c01b459d27. * (can't repro locally) just athena should be fine * bs1 * bs2 * bs3 * bs4 * bs5 * wrong way * no controls * no car * no board * controls? * crazy -common * Revert "crazy -common" This reverts commit 02365d712b3d09cab1893cce2261a4b418bb3851. * test athena * test athena 2 * test athena 3 * test athena 4 * test athena 5 * test athena 6 * test athena 7 * test athena 8 * test athena 9 * ?? * in one commit * common? * car and board * -controls -board * random-order * no board * revert * car/tests * least likely * try * try 2 * draft * draft * so much better * cmt * use randomly * not needed here * directly modify option works * bb * test time * Revert "test time" This reverts commit 2c5caabe2b470b47b7322e37800680b92773fccc. * tmut * i concur * revert old-commit-hash: ac83318ac4f54c46deef02c69d46c549bd741cb3 --- conftest.py | 6 ++++++ poetry.lock | 4 ++-- pyproject.toml | 2 +- selfdrive/car/tests/test_models.py | 11 +++++------ 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/conftest.py b/conftest.py index 285d8b0238..3c566e3672 100644 --- a/conftest.py +++ b/conftest.py @@ -6,6 +6,12 @@ from openpilot.common.prefix import OpenpilotPrefix from openpilot.system.hardware import TICI +def pytest_sessionstart(session): + # TODO: fix tests and enable test order randomization + if session.config.pluginmanager.hasplugin('randomly'): + session.config.option.randomly_reorganize = False + + @pytest.fixture(scope="function", autouse=True) def openpilot_function_fixture(): starting_env = dict(os.environ) diff --git a/poetry.lock b/poetry.lock index 4064417188..da14fcc1eb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1c6779df1d2ef415664f495c1b06687d06d93588009ca4d1967ad391b58cc1d4 -size 438605 +oid sha256:e7e3be3454e2f4a74667fce67eda6068e5296f9dad824fc2ea8c21566dc77b29 +size 438272 diff --git a/pyproject.toml b/pyproject.toml index 0116cb4c5e..c6a13c4b86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -151,7 +151,7 @@ pytest-subtests = "*" pytest-xdist = "*" pytest-timeout = "*" pytest-timeouts = "*" -pytest-random-order = "*" +pytest-randomly = "*" ruff = "*" scipy = "*" sphinx = "*" diff --git a/selfdrive/car/tests/test_models.py b/selfdrive/car/tests/test_models.py index 0ae104f9f5..aaaa6b9d79 100755 --- a/selfdrive/car/tests/test_models.py +++ b/selfdrive/car/tests/test_models.py @@ -3,6 +3,7 @@ import capnp import os import importlib import pytest +import random import unittest from collections import defaultdict, Counter from typing import List, Optional, Tuple @@ -49,12 +50,10 @@ def get_test_cases() -> List[Tuple[str, Optional[CarTestRoute]]]: with open(os.path.join(BASEDIR, INTERNAL_SEG_LIST), "r") as f: seg_list = f.read().splitlines() - cnt = INTERNAL_SEG_CNT or len(seg_list) - seg_list_iter = iter(seg_list[:cnt]) - - for platform in seg_list_iter: - platform = platform[2:] # get rid of comment - segment_name = SegmentName(next(seg_list_iter)) + seg_list_grouped = [(platform[2:], segment) for platform, segment in zip(seg_list[::2], seg_list[1::2], strict=True)] + seg_list_grouped = random.sample(seg_list_grouped, INTERNAL_SEG_CNT or len(seg_list_grouped)) + for platform, segment in seg_list_grouped: + segment_name = SegmentName(segment) test_cases.append((platform, CarTestRoute(segment_name.route_name.canonical_name, platform, segment=segment_name.segment_num))) return test_cases