run unit tests in a random order (#29799)

* run unit tests in a random order by default

* bump panda

* fix startup

* re-add random order plugin

---------

Co-authored-by: Justin Newberry <justin@comma.ai>
old-commit-hash: b483efe764

poetry.lock

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2c58f799c5ca463de41f69bd227238e2c03911501d90e656288fcaba07506da5
+oid sha256:6d893d3b28752a00201d9352630b094fe4e0e07fbd2921775485929830ff89ad
-size 449347
+size 445868

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.pytest.ini_options]
 minversion = "6.0"
-addopts = "--ignore=openpilot/ --ignore=cereal/ --ignore=opendbc/ --ignore=panda/ --ignore=rednose_repo/ --ignore=tinygrad_repo/ --ignore=laika_repo/ -Werror --strict-config --strict-markers --durations=10"
+addopts = "--ignore=openpilot/ --ignore=cereal/ --ignore=opendbc/ --ignore=panda/ --ignore=rednose_repo/ --ignore=tinygrad_repo/ --ignore=laika_repo/ -Werror --strict-config --strict-markers --durations=10 --random-order"
 #cpp_files = "test_*" # uncomment when agnos has pytest-cpp and remove from CI
 python_files = "test_*.py"
 #timeout = "30"  # you get this long by default
@@ -152,6 +152,7 @@ pytest-subtests = "*"
 pytest-xdist = "*"
 pytest-timeout = "*"
 pytest-timeouts = "*"
+pytest-random-order = "*"
 ruff = "*"
 scipy = "*"
 sphinx = "*"

selfdrive/controls/tests/test_startup.py

@@ -38,6 +38,12 @@ CX5_FW_VERSIONS = [
 
 class TestStartup(unittest.TestCase):
 
+  def setUp(self):
+    self.params = Params()
+    self.params.clear_all()
+    self.params.put_bool("Passive", False)
+    self.params.put_bool("OpenpilotEnabledToggle", True)
+
   @parameterized.expand([
     # TODO: test EventName.startup for release branches
 
@@ -68,11 +74,6 @@ class TestStartup(unittest.TestCase):
     controls_sock = messaging.sub_sock("controlsState")
     pm = messaging.PubMaster(['can', 'pandaStates'])
 
-    params = Params()
-    params.clear_all()
-    params.put_bool("Passive", False)
-    params.put_bool("OpenpilotEnabledToggle", True)
-
     # Build capnn version of FW array
     if fw_versions is not None:
       car_fw = []
@@ -90,7 +91,7 @@ class TestStartup(unittest.TestCase):
         car_fw.append(f)
       cp.carVin = "1" * 17
       cp.carFw = car_fw
-      params.put("CarParamsCache", cp.to_bytes())
+      self.params.put("CarParamsCache", cp.to_bytes())
 
     time.sleep(2) # wait for controlsd to be ready
 
@@ -109,8 +110,8 @@ class TestStartup(unittest.TestCase):
 
     for _ in range(1000):
       # controlsd waits for boardd to echo back that it has changed the multiplexing mode
-      if not params.get_bool("ObdMultiplexingChanged"):
+      if not self.params.get_bool("ObdMultiplexingChanged"):
-        params.put_bool("ObdMultiplexingChanged", True)
+        self.params.put_bool("ObdMultiplexingChanged", True)
 
       msgs = [[addr, 0, b'\x00'*length, 0] for addr, length in finger.items()]
       pm.send('can', can_list_to_can_capnp(msgs))