test_models: randomize internal segment list (#30653)

* randomize internal seg list segments

* random

* pytest-randomly sets random.seed to a consistent value for all the workers/processes

* noeol

* update

* Revert "update"

This reverts commit aff9a69c4e5e3934deebaa33986b42f44b55b002.

* lock

* don't randomize by default

* remove random-order

* strict

* random

* one fix

* test

* does nothing

* rm tests

* Revert "rm tests"

This reverts commit b548e3fcd48e60538695506888d863c01b459d27.

* (can't repro locally) just athena should be fine

* bs1

* bs2

* bs3

* bs4

* bs5

* wrong way

* no controls

* no car

* no board

* controls?

* crazy -common

* Revert "crazy -common"

This reverts commit 02365d712b3d09cab1893cce2261a4b418bb3851.

* test athena

* test athena 2

* test athena 3

* test athena 4

* test athena 5

* test athena 6

* test athena 7

* test athena 8

* test athena 9

* ??

* in one commit

* common?

* car and board

* -controls -board

* random-order

* no board

* revert

* car/tests

* least likely

* try

* try 2

* draft

* draft

* so much better

* cmt

* use randomly

* not needed here

* directly modify option works

* bb

* test time

* Revert "test time"

This reverts commit 2c5caabe2b470b47b7322e37800680b92773fccc.

* tmut

* i concur

* revert

conftest.py

@@ -6,6 +6,12 @@ from openpilot.common.prefix import OpenpilotPrefix
 from openpilot.system.hardware import TICI
 
 
+def pytest_sessionstart(session):
+  # TODO: fix tests and enable test order randomization
+  if session.config.pluginmanager.hasplugin('randomly'):
+    session.config.option.randomly_reorganize = False
+
+
 @pytest.fixture(scope="function", autouse=True)
 def openpilot_function_fixture():
   starting_env = dict(os.environ)

poetry.lock

@@ -3309,8 +3309,6 @@ files = [
     {file = "pygame-2.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e24d05184e4195fe5ebcdce8b18ecb086f00182b9ae460a86682d312ce8d31f"},
     {file = "pygame-2.5.2-cp311-cp311-win32.whl", hash = "sha256:f02c1c7505af18d426d355ac9872bd5c916b27f7b0fe224749930662bea47a50"},
     {file = "pygame-2.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:6d58c8cf937815d3b7cdc0fa9590c5129cb2c9658b72d00e8a4568dea2ff1d42"},
-    {file = "pygame-2.5.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1a2a43802bb5e89ce2b3b775744e78db4f9a201bf8d059b946c61722840ceea8"},
-    {file = "pygame-2.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1c289f2613c44fe70a1e40769de4a49c5ab5a29b9376f1692bb1a15c9c1c9bfa"},
     {file = "pygame-2.5.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:074aa6c6e110c925f7f27f00c7733c6303407edc61d738882985091d1eb2ef17"},
     {file = "pygame-2.5.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe0228501ec616779a0b9c4299e837877783e18df294dd690b9ab0eed3d8aaab"},
     {file = "pygame-2.5.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31648d38ecdc2335ffc0e38fb18a84b3339730521505dac68514f83a1092e3f4"},
@@ -3719,18 +3717,18 @@ colorama = "*"
 pytest = ">=7.0"
 
 [[package]]
-name = "pytest-random-order"
+name = "pytest-randomly"
-version = "1.1.0"
+version = "3.15.0"
-description = "Randomise the order in which pytest tests are run with some control over the randomness"
+description = "Pytest plugin to randomly order tests and control random.seed."
 optional = false
-python-versions = ">=3.5.0"
+python-versions = ">=3.8"
 files = [
-    {file = "pytest-random-order-1.1.0.tar.gz", hash = "sha256:dbe6debb9353a7af984cc9eddbeb3577dd4dbbcc1529a79e3d21f68ed9b45605"},
+    {file = "pytest_randomly-3.15.0-py3-none-any.whl", hash = "sha256:0516f4344b29f4e9cdae8bce31c4aeebf59d0b9ef05927c33354ff3859eeeca6"},
-    {file = "pytest_random_order-1.1.0-py3-none-any.whl", hash = "sha256:6cb1e59ab0f798bb0c3488c11ae0c70d7d3340306a466d28b28ccd8ef8c20b7e"},
+    {file = "pytest_randomly-3.15.0.tar.gz", hash = "sha256:b908529648667ba5e54723088edd6f82252f540cc340d748d1fa985539687047"},
 ]
 
 [package.dependencies]
-pytest = ">=3.0.0"
+pytest = "*"
 
 [[package]]
 name = "pytest-subtests"
@@ -5011,4 +5009,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
 [metadata]
 lock-version = "2.0"
 python-versions = "~3.11"
-content-hash = "f0abc0f53443c3b98eb06fc4dd0ff736e39197bc68c79db9aeffb4736657b676"
+content-hash = "7f7244a89280a41937066ebf80aa64fcb375852484c9d034ea7f2de3eae4d90a"

pyproject.toml

@@ -151,7 +151,7 @@ pytest-subtests = "*"
 pytest-xdist = "*"
 pytest-timeout = "*"
 pytest-timeouts = "*"
-pytest-random-order = "*"
+pytest-randomly = "*"
 ruff = "*"
 scipy = "*"
 sphinx = "*"

selfdrive/car/tests/test_models.py

@@ -3,6 +3,7 @@ import capnp
 import os
 import importlib
 import pytest
+import random
 import unittest
 from collections import defaultdict, Counter
 from typing import List, Optional, Tuple
@@ -49,12 +50,10 @@ def get_test_cases() -> List[Tuple[str, Optional[CarTestRoute]]]:
     with open(os.path.join(BASEDIR, INTERNAL_SEG_LIST), "r") as f:
       seg_list = f.read().splitlines()
 
-    cnt = INTERNAL_SEG_CNT or len(seg_list)
+    seg_list_grouped = [(platform[2:], segment) for platform, segment in zip(seg_list[::2], seg_list[1::2], strict=True)]
-    seg_list_iter = iter(seg_list[:cnt])
+    seg_list_grouped = random.sample(seg_list_grouped, INTERNAL_SEG_CNT or len(seg_list_grouped))
-
+    for platform, segment in seg_list_grouped:
-    for platform in seg_list_iter:
+      segment_name = SegmentName(segment)
-      platform = platform[2:]  # get rid of comment
-      segment_name = SegmentName(next(seg_list_iter))
       test_cases.append((platform, CarTestRoute(segment_name.route_name.canonical_name, platform,
                                                 segment=segment_name.segment_num)))
   return test_cases