test_models: randomize internal segment list (#30653)

* randomize internal seg list segments

* random

* pytest-randomly sets random.seed to a consistent value for all the workers/processes

* noeol

* update

* Revert "update"

This reverts commit aff9a69c4e.

* lock

* don't randomize by default

* remove random-order

* strict

* random

* one fix

* test

* does nothing

* rm tests

* Revert "rm tests"

This reverts commit b548e3fcd4.

* (can't repro locally) just athena should be fine

* bs1

* bs2

* bs3

* bs4

* bs5

* wrong way

* no controls

* no car

* no board

* controls?

* crazy -common

* Revert "crazy -common"

This reverts commit 02365d712b.

* test athena

* test athena 2

* test athena 3

* test athena 4

* test athena 5

* test athena 6

* test athena 7

* test athena 8

* test athena 9

* ??

* in one commit

* common?

* car and board

* -controls -board

* random-order

* no board

* revert

* car/tests

* least likely

* try

* try 2

* draft

* draft

* so much better

* cmt

* use randomly

* not needed here

* directly modify option works

* bb

* test time

* Revert "test time"

This reverts commit 2c5caabe2b.

* tmut

* i concur

* revert
old-commit-hash: ac83318ac4

conftest.py

@@ -6,6 +6,12 @@ from openpilot.common.prefix import OpenpilotPrefix
 from openpilot.system.hardware import TICI
 
 
+def pytest_sessionstart(session):
+  # TODO: fix tests and enable test order randomization
+  if session.config.pluginmanager.hasplugin('randomly'):
+    session.config.option.randomly_reorganize = False
+
+
 @pytest.fixture(scope="function", autouse=True)
 def openpilot_function_fixture():
   starting_env = dict(os.environ)

poetry.lock

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c6779df1d2ef415664f495c1b06687d06d93588009ca4d1967ad391b58cc1d4
-size 438605
+oid sha256:e7e3be3454e2f4a74667fce67eda6068e5296f9dad824fc2ea8c21566dc77b29
+size 438272

pyproject.toml

@@ -151,7 +151,7 @@ pytest-subtests = "*"
 pytest-xdist = "*"
 pytest-timeout = "*"
 pytest-timeouts = "*"
-pytest-random-order = "*"
+pytest-randomly = "*"
 ruff = "*"
 scipy = "*"
 sphinx = "*"

selfdrive/car/tests/test_models.py

@@ -3,6 +3,7 @@ import capnp
 import os
 import importlib
 import pytest
+import random
 import unittest
 from collections import defaultdict, Counter
 from typing import List, Optional, Tuple
@@ -49,12 +50,10 @@ def get_test_cases() -> List[Tuple[str, Optional[CarTestRoute]]]:
     with open(os.path.join(BASEDIR, INTERNAL_SEG_LIST), "r") as f:
       seg_list = f.read().splitlines()
 
-    cnt = INTERNAL_SEG_CNT or len(seg_list)
-    seg_list_iter = iter(seg_list[:cnt])
-
-    for platform in seg_list_iter:
-      platform = platform[2:]  # get rid of comment
-      segment_name = SegmentName(next(seg_list_iter))
+    seg_list_grouped = [(platform[2:], segment) for platform, segment in zip(seg_list[::2], seg_list[1::2], strict=True)]
+    seg_list_grouped = random.sample(seg_list_grouped, INTERNAL_SEG_CNT or len(seg_list_grouped))
+    for platform, segment in seg_list_grouped:
+      segment_name = SegmentName(segment)
       test_cases.append((platform, CarTestRoute(segment_name.route_name.canonical_name, platform,
                                                 segment=segment_name.segment_num)))
   return test_cases