Pytest: tici mark for skipping on-device tests (#30412)

* mark tici

* not those ones

* missed that one

* add those

* add pypoetry

* oops
old-commit-hash: aec7cea30d

conftest.py

@@ -2,6 +2,7 @@ import os
 import pytest
 
 from openpilot.common.prefix import OpenpilotPrefix
+from openpilot.system.hardware import TICI
 
 
 @pytest.fixture(scope="function", autouse=True)
@@ -31,3 +32,10 @@ def openpilot_class_fixture():
 
   os.environ.clear()
   os.environ.update(starting_env)
+
+
+def pytest_collection_modifyitems(config, items):
+  skipper = pytest.mark.skip(reason="Skipping tici test on PC")
+  for item in items:
+    if not TICI and "tici" in item.keywords:
+      item.add_marker(skipper)

pyproject.toml

@@ -5,7 +5,8 @@ addopts = "--ignore=openpilot/ --ignore=cereal/ --ignore=opendbc/ --ignore=panda
 python_files = "test_*.py"
 #timeout = "30"  # you get this long by default
 markers = [
-  "slow: tests that take awhile to run and can be skipped with -m 'not slow'"
+  "slow: tests that take awhile to run and can be skipped with -m 'not slow'",
+  "tici: tests that are only meant to run on the C3/C3X",
 ]
 testpaths = [
   "common",

release/files_common

@@ -6,6 +6,7 @@ launch_openpilot.sh
 
 Jenkinsfile
 SConstruct
+pyproject.toml
 
 README.md
 RELEASES.md

selfdrive/boardd/tests/test_pandad.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import os
+import pytest
 import time
 import unittest
 
@@ -8,17 +9,16 @@ from cereal import log
 from openpilot.common.gpio import gpio_set, gpio_init
 from panda import Panda, PandaDFU, PandaProtocolMismatch
 from openpilot.selfdrive.manager.process_config import managed_processes
-from openpilot.system.hardware import HARDWARE, PC
+from openpilot.system.hardware import HARDWARE
 from openpilot.system.hardware.tici.pins import GPIO
 
 HERE = os.path.dirname(os.path.realpath(__file__))
 
 
+@pytest.mark.tici
 class TestPandad(unittest.TestCase):
 
   def setUp(self):
-    if PC:
-      raise unittest.SkipTest("needs a panda")
     # ensure panda is up
     if len(Panda.list()) == 0:
       self._run_test(60)

selfdrive/manager/test/test_manager.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import os
+import pytest
 import signal
 import time
 import unittest
@@ -17,6 +18,7 @@ MAX_STARTUP_TIME = 3
 BLACKLIST_PROCS = ['manage_athenad', 'pandad', 'pigeond']
 
 
+@pytest.mark.tici
 class TestManager(unittest.TestCase):
   def setUp(self):
     os.environ['PASSIVE'] = '0'

selfdrive/modeld/tests/test_modeld.py

@@ -13,6 +13,7 @@ from openpilot.selfdrive.test.process_replay.vision_meta import meta_from_camera
 IMG = np.zeros(int(tici_f_frame_size[0]*tici_f_frame_size[1]*(3/2)), dtype=np.uint8)
 IMG_BYTES = IMG.flatten().tobytes()
 
+
 class TestModeld(unittest.TestCase):
 
   def setUp(self):

selfdrive/navd/tests/test_map_renderer.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import numpy as np
 import os
+import pytest
 import unittest
 import requests
 import threading
@@ -10,7 +11,6 @@ import cereal.messaging as messaging
 from typing import Any
 from cereal.visionipc import VisionIpcClient, VisionStreamType
 from openpilot.selfdrive.manager.process_config import managed_processes
-from openpilot.system.hardware import TICI
 
 LLK_DECIMATION = 10
 CACHE_PATH = "/data/mbgl-cache-navd.db"
@@ -189,10 +189,8 @@ class TestMapRenderer(unittest.TestCase):
 
     self._run_test(True, LOCATION2_REPEATED)
 
+  @pytest.mark.tici
   def test_render_time_distribution(self):
-    if not TICI:
-      raise unittest.SkipTest
-
     self._setup_test()
     # from location1 -> location2 and back
     locations = np.array([*np.linspace(LOCATION1, LOCATION2, 2000), *np.linspace(LOCATION2, LOCATION1, 2000)]).tolist()

selfdrive/test/test_onroad.py

@@ -5,6 +5,7 @@ import json
 import os
 import pathlib
 import psutil
+import pytest
 import shutil
 import subprocess
 import time
@@ -98,6 +99,7 @@ def cputime_total(ct):
   return ct.cpuUser + ct.cpuSystem + ct.cpuChildrenUser + ct.cpuChildrenSystem
 
 
+@pytest.mark.tici
 class TestOnroad(unittest.TestCase):
 
   @classmethod

selfdrive/test/test_time_to_onroad.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import os
+import pytest
 import time
 import subprocess
 
@@ -9,6 +10,7 @@ from openpilot.common.timeout import Timeout
 from openpilot.selfdrive.test.helpers import set_params_enabled
 
 
+@pytest.mark.tici
 def test_time_to_onroad():
   # launch
   set_params_enabled()

selfdrive/test/test_updated.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import datetime
 import os
+import pytest
 import time
 import tempfile
 import unittest
@@ -13,6 +14,7 @@ from openpilot.common.basedir import BASEDIR
 from openpilot.common.params import Params
 
 
+@pytest.mark.tici
 class TestUpdated(unittest.TestCase):
 
   def setUp(self):

system/camerad/test/test_camerad.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import pytest
 import time
 import unittest
 import numpy as np
@@ -8,7 +9,6 @@ import cereal.messaging as messaging
 from cereal import log
 from cereal.services import SERVICE_LIST
 from openpilot.selfdrive.manager.process_config import managed_processes
-from openpilot.system.hardware import TICI
 
 TEST_TIMESPAN = 30
 LAG_FRAME_TOLERANCE = {log.FrameData.ImageSensor.ar0231: 0.5,  # ARs use synced pulses for frame starts
@@ -19,12 +19,10 @@ FRAME_DELTA_TOLERANCE = {log.FrameData.ImageSensor.ar0231: 1.0,
 CAMERAS = ('roadCameraState', 'driverCameraState', 'wideRoadCameraState')
 
 
+@pytest.mark.tici
 class TestCamerad(unittest.TestCase):
   @classmethod
   def setUpClass(cls):
-    if not TICI:
-      raise unittest.SkipTest
-
     # run camerad and record logs
     managed_processes['camerad'].start()
     time.sleep(3)

system/hardware/tici/tests/test_hardware.py

@@ -1,20 +1,19 @@
 #!/usr/bin/env python3
+import pytest
 import time
 import unittest
 import numpy as np
 
-from openpilot.system.hardware import TICI
 from openpilot.system.hardware.tici.hardware import Tici
 
 HARDWARE = Tici()
 
+
+@pytest.mark.tici
 class TestHardware(unittest.TestCase):
 
   @classmethod
   def setUpClass(cls):
-    if not TICI:
-      raise unittest.SkipTest
-
     HARDWARE.initialize_hardware()
     HARDWARE.set_power_save(False)
 

system/hardware/tici/tests/test_power_draw.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import pytest
 import unittest
 import time
 import threading
@@ -9,7 +10,7 @@ from typing import List
 
 import cereal.messaging as messaging
 from cereal.services import SERVICE_LIST
-from openpilot.system.hardware import HARDWARE, TICI
+from openpilot.system.hardware import HARDWARE
 from openpilot.system.hardware.tici.power_monitor import get_power
 from openpilot.selfdrive.manager.process_config import managed_processes
 from openpilot.selfdrive.manager.manager import manager_cleanup
@@ -44,13 +45,9 @@ def send_llk_msg(done):
     time.sleep(1/20.)
 
 
+@pytest.mark.tici
 class TestPowerDraw(unittest.TestCase):
 
-  @classmethod
-  def setUpClass(cls):
-    if not TICI:
-      raise unittest.SkipTest
-
   def setUp(self):
     HARDWARE.initialize_hardware()
     HARDWARE.set_power_save(False)

system/loggerd/tests/test_encoder.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import math
 import os
+import pytest
 import random
 import shutil
 import subprocess
@@ -31,14 +32,9 @@ CAMERAS = [
 FILE_SIZE_TOLERANCE = 0.5
 
 
+@pytest.mark.tici # TODO: all of loggerd should work on PC
 class TestEncoder(unittest.TestCase):
 
-  # TODO: all of loggerd should work on PC
-  @classmethod
-  def setUpClass(cls):
-    if not TICI:
-      raise unittest.SkipTest
-
   def setUp(self):
     self._clear_logs()
     os.environ["LOGGERD_TEST"] = "1"

system/sensord/rawgps/test_rawgps.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import os
+import pytest
 import json
 import time
 import datetime
@@ -7,19 +8,16 @@ import unittest
 import subprocess
 
 import cereal.messaging as messaging
-from openpilot.system.hardware import TICI
 from openpilot.system.sensord.rawgps.rawgpsd import at_cmd, wait_for_modem
 from openpilot.selfdrive.manager.process_config import managed_processes
 
 GOOD_SIGNAL = bool(int(os.getenv("GOOD_SIGNAL", '0')))
 
 
+@pytest.mark.tici
 class TestRawgpsd(unittest.TestCase):
   @classmethod
   def setUpClass(cls):
-    if not TICI:
-      raise unittest.SkipTest
-
     os.system("sudo systemctl start systemd-resolved")
     os.system("sudo systemctl restart ModemManager lte")
     wait_for_modem()

system/sensord/tests/test_pigeond.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import pytest
 import time
 import unittest
 
@@ -7,16 +8,12 @@ from cereal.services import SERVICE_LIST
 from openpilot.common.gpio import gpio_read
 from openpilot.selfdrive.test.helpers import with_processes
 from openpilot.selfdrive.manager.process_config import managed_processes
-from openpilot.system.hardware import TICI
 from openpilot.system.hardware.tici.pins import GPIO
 
 
 # TODO: test TTFF when we have good A-GNSS
+@pytest.mark.tici
 class TestPigeond(unittest.TestCase):
-  @classmethod
-  def setUpClass(cls):
-    if not TICI:
-      raise unittest.SkipTest
 
   def tearDown(self):
     managed_processes['pigeond'].stop()

system/sensord/tests/test_sensord.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import os
+import pytest
 import time
 import unittest
 import numpy as np
@@ -10,7 +11,6 @@ from cereal import log
 from cereal.services import SERVICE_LIST
 from openpilot.common.gpio import get_irqs_for_action
 from openpilot.common.timeout import Timeout
-from openpilot.system.hardware import TICI
 from openpilot.selfdrive.manager.process_config import managed_processes
 
 BMX = {
@@ -98,12 +98,10 @@ def read_sensor_events(duration_sec):
 
   return {k: v for k, v in events.items() if len(v) > 0}
 
+@pytest.mark.tici
 class TestSensord(unittest.TestCase):
   @classmethod
   def setUpClass(cls):
-    if not TICI:
-      raise unittest.SkipTest
-
     # enable LSM self test
     os.environ["LSM_SELF_TEST"] = "1"
 

tools/gpstest/test_gps.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import pytest
 import time
 import unittest
 import struct
@@ -6,7 +7,6 @@ import struct
 from openpilot.common.params import Params
 import cereal.messaging as messaging
 import openpilot.system.sensord.pigeond as pd
-from openpilot.system.hardware import TICI
 from openpilot.selfdrive.test.helpers import with_processes
 
 
@@ -107,12 +107,10 @@ def verify_time_to_first_fix(pigeon):
   assert ttff < 40, f"Time to first fix > 40s, {ttff}"
 
 
+@pytest.mark.tici
 class TestGPS(unittest.TestCase):
   @classmethod
   def setUpClass(cls):
-    if not TICI:
-      raise unittest.SkipTest
-
     ublox_available = Params().get_bool("UbloxAvailable")
     if not ublox_available:
       raise unittest.SkipTest

tools/gpstest/test_gps_qcom.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python3
+import pytest
 import time
 import unittest
 import subprocess as sp
 
 from openpilot.common.params import Params
-from openpilot.system.hardware import TICI
 import cereal.messaging as messaging
 from openpilot.selfdrive.manager.process_config import managed_processes
 
@@ -30,12 +30,10 @@ def wait_for_location(socket, timeout):
     continue
 
 
+@pytest.mark.tici
 class TestGPS(unittest.TestCase):
   @classmethod
   def setUpClass(cls):
-    if not TICI:
-      raise unittest.SkipTest
-
     ublox_available = Params().get_bool("UbloxAvailable")
     if ublox_available:
       raise unittest.SkipTest