Pytest: load by group by default (#30466)

* wip

* use a class marker

* this one should be grouped too

conftest.py

@@ -37,8 +37,21 @@ def openpilot_class_fixture():
   os.environ.update(starting_env)
 
 
+@pytest.hookimpl(tryfirst=True)
 def pytest_collection_modifyitems(config, items):
   skipper = pytest.mark.skip(reason="Skipping tici test on PC")
   for item in items:
     if not TICI and "tici" in item.keywords:
       item.add_marker(skipper)
+
+    if "xdist_group_class_property" in item.keywords:
+      class_property = item.get_closest_marker('xdist_group_class_property').args[0]
+      item.add_marker(pytest.mark.xdist_group(getattr(item.cls, class_property)))
+
+
+@pytest.hookimpl(trylast=True)
+def pytest_configure(config):
+    config_line = (
+        "xdist_group_class_property: group tests by a property of the class that contains them"
+    )
+    config.addinivalue_line("markers", config_line)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.pytest.ini_options]
 minversion = "6.0"
-addopts = "--ignore=openpilot/ --ignore=cereal/ --ignore=opendbc/ --ignore=panda/ --ignore=rednose_repo/ --ignore=tinygrad_repo/ --ignore=laika_repo/ -Werror --strict-config --strict-markers --durations=10 -n auto --dist=loadscope"
+addopts = "--ignore=openpilot/ --ignore=cereal/ --ignore=opendbc/ --ignore=panda/ --ignore=rednose_repo/ --ignore=tinygrad_repo/ --ignore=laika_repo/ -Werror --strict-config --strict-markers --durations=10 -n auto --dist=loadgroup"
 cpp_files = "test_*"
 python_files = "test_*.py"
 #timeout = "30"  # you get this long by default

selfdrive/car/tests/test_models.py

@@ -50,7 +50,7 @@ def get_test_cases() -> List[Tuple[str, Optional[CarTestRoute]]]:
 
     for i, c in enumerate(sorted(all_known_cars())):
       if i % NUM_JOBS == JOB_ID:
-        test_cases.extend(sorted((c, r) for r in routes_by_car.get(c, (None,))))
+        test_cases.extend(sorted((c.value, r) for r in routes_by_car.get(c, (None,))))
 
   else:
     with open(os.path.join(BASEDIR, INTERNAL_SEG_LIST), "r") as f:
@@ -385,6 +385,7 @@ class TestCarModelBase(unittest.TestCase):
 
 
 @parameterized_class(('car_model', 'test_route'), get_test_cases())
+@pytest.mark.xdist_group_class_property('car_model')
 class TestCarModel(TestCarModelBase):
   pass
 

selfdrive/locationd/test/test_locationd_scenarios.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import pytest
 import unittest
 import numpy as np
 from collections import defaultdict
@@ -97,6 +98,7 @@ def run_scenarios(scenario, logs):
   return get_select_fields_data(logs), get_select_fields_data(replayed_logs)
 
 
+@pytest.mark.xdist_group("test_locationd_scenarios")
 class TestLocationdScenarios(unittest.TestCase):
   """
   Test locationd with different scenarios. In all these scenarios, we expect the following:

selfdrive/test/process_replay/test_processes.py

@@ -63,6 +63,7 @@ ALL_CARS = sorted({car for car, _ in segments})
 
 @pytest.mark.slow
 @parameterized_class(('case_name', 'segment'), segments)
+@pytest.mark.xdist_group_class_property('case_name')
 class TestCarProcessReplay(TestProcessReplayDiffBase):
   """
   Runs a replay diff on a segment for each car.