From 3027a428cbcd3334239e5ad4fdfb1cf749fa8a7f Mon Sep 17 00:00:00 2001
From: Justin Newberry <justin@comma.ai>
Date: Tue, 7 Nov 2023 20:35:44 -0500
Subject: [PATCH] Pytest: tici mark for skipping on-device tests (#30412)

* mark tici

* not those ones

* missed that one

* add those

* add pypoetry

* oops
old-commit-hash: aec7cea30d48dc8915eea38c86de7d4effa0548d
---
 conftest.py                                   | 8 ++++++++
 pyproject.toml                                | 3 ++-
 release/files_common                          | 1 +
 selfdrive/boardd/tests/test_pandad.py         | 6 +++---
 selfdrive/manager/test/test_manager.py        | 2 ++
 selfdrive/modeld/tests/test_modeld.py         | 1 +
 selfdrive/navd/tests/test_map_renderer.py     | 6 ++----
 selfdrive/test/test_onroad.py                 | 2 ++
 selfdrive/test/test_time_to_onroad.py         | 2 ++
 selfdrive/test/test_updated.py                | 2 ++
 system/camerad/test/test_camerad.py           | 6 ++----
 system/hardware/tici/tests/test_hardware.py   | 7 +++----
 system/hardware/tici/tests/test_power_draw.py | 9 +++------
 system/loggerd/tests/test_encoder.py          | 8 ++------
 system/sensord/rawgps/test_rawgps.py          | 6 ++----
 system/sensord/tests/test_pigeond.py          | 7 ++-----
 system/sensord/tests/test_sensord.py          | 6 ++----
 tools/gpstest/test_gps.py                     | 6 ++----
 tools/gpstest/test_gps_qcom.py                | 6 ++----
 19 files changed, 45 insertions(+), 49 deletions(-)

diff --git a/conftest.py b/conftest.py
index ee46436d3c..085ad5b28a 100644
--- a/conftest.py
+++ b/conftest.py
@@ -2,6 +2,7 @@ import os
 import pytest
 
 from openpilot.common.prefix import OpenpilotPrefix
+from openpilot.system.hardware import TICI
 
 
 @pytest.fixture(scope="function", autouse=True)
@@ -31,3 +32,10 @@ def openpilot_class_fixture():
 
   os.environ.clear()
   os.environ.update(starting_env)
+
+
+def pytest_collection_modifyitems(config, items):
+  skipper = pytest.mark.skip(reason="Skipping tici test on PC")
+  for item in items:
+    if not TICI and "tici" in item.keywords:
+      item.add_marker(skipper)
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index c83669898f..71b677fbb7 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -5,7 +5,8 @@ addopts = "--ignore=openpilot/ --ignore=cereal/ --ignore=opendbc/ --ignore=panda
 python_files = "test_*.py"
 #timeout = "30"  # you get this long by default
 markers = [
-  "slow: tests that take awhile to run and can be skipped with -m 'not slow'"
+  "slow: tests that take awhile to run and can be skipped with -m 'not slow'",
+  "tici: tests that are only meant to run on the C3/C3X",
 ]
 testpaths = [
   "common",
diff --git a/release/files_common b/release/files_common
index 9076f440d5..5a076f2b1a 100644
--- a/release/files_common
+++ b/release/files_common
@@ -6,6 +6,7 @@ launch_openpilot.sh
 
 Jenkinsfile
 SConstruct
+pyproject.toml
 
 README.md
 RELEASES.md
diff --git a/selfdrive/boardd/tests/test_pandad.py b/selfdrive/boardd/tests/test_pandad.py
index c1bf22f146..4036766faa 100755
--- a/selfdrive/boardd/tests/test_pandad.py
+++ b/selfdrive/boardd/tests/test_pandad.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import os
+import pytest
 import time
 import unittest
 
@@ -8,17 +9,16 @@ from cereal import log
 from openpilot.common.gpio import gpio_set, gpio_init
 from panda import Panda, PandaDFU, PandaProtocolMismatch
 from openpilot.selfdrive.manager.process_config import managed_processes
-from openpilot.system.hardware import HARDWARE, PC
+from openpilot.system.hardware import HARDWARE
 from openpilot.system.hardware.tici.pins import GPIO
 
 HERE = os.path.dirname(os.path.realpath(__file__))
 
 
+@pytest.mark.tici
 class TestPandad(unittest.TestCase):
 
   def setUp(self):
-    if PC:
-      raise unittest.SkipTest("needs a panda")
     # ensure panda is up
     if len(Panda.list()) == 0:
       self._run_test(60)
diff --git a/selfdrive/manager/test/test_manager.py b/selfdrive/manager/test/test_manager.py
index d4d4e64f78..0fa186baff 100755
--- a/selfdrive/manager/test/test_manager.py
+++ b/selfdrive/manager/test/test_manager.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import os
+import pytest
 import signal
 import time
 import unittest
@@ -17,6 +18,7 @@ MAX_STARTUP_TIME = 3
 BLACKLIST_PROCS = ['manage_athenad', 'pandad', 'pigeond']
 
 
+@pytest.mark.tici
 class TestManager(unittest.TestCase):
   def setUp(self):
     os.environ['PASSIVE'] = '0'
diff --git a/selfdrive/modeld/tests/test_modeld.py b/selfdrive/modeld/tests/test_modeld.py
index 7ae6d5308f..e010f6cfd6 100755
--- a/selfdrive/modeld/tests/test_modeld.py
+++ b/selfdrive/modeld/tests/test_modeld.py
@@ -13,6 +13,7 @@ from openpilot.selfdrive.test.process_replay.vision_meta import meta_from_camera
 IMG = np.zeros(int(tici_f_frame_size[0]*tici_f_frame_size[1]*(3/2)), dtype=np.uint8)
 IMG_BYTES = IMG.flatten().tobytes()
 
+
 class TestModeld(unittest.TestCase):
 
   def setUp(self):
diff --git a/selfdrive/navd/tests/test_map_renderer.py b/selfdrive/navd/tests/test_map_renderer.py
index c4c9785cb1..dbd6add995 100755
--- a/selfdrive/navd/tests/test_map_renderer.py
+++ b/selfdrive/navd/tests/test_map_renderer.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import numpy as np
 import os
+import pytest
 import unittest
 import requests
 import threading
@@ -10,7 +11,6 @@ import cereal.messaging as messaging
 from typing import Any
 from cereal.visionipc import VisionIpcClient, VisionStreamType
 from openpilot.selfdrive.manager.process_config import managed_processes
-from openpilot.system.hardware import TICI
 
 LLK_DECIMATION = 10
 CACHE_PATH = "/data/mbgl-cache-navd.db"
@@ -189,10 +189,8 @@ class TestMapRenderer(unittest.TestCase):
 
     self._run_test(True, LOCATION2_REPEATED)
 
+  @pytest.mark.tici
   def test_render_time_distribution(self):
-    if not TICI:
-      raise unittest.SkipTest
-
     self._setup_test()
     # from location1 -> location2 and back
     locations = np.array([*np.linspace(LOCATION1, LOCATION2, 2000), *np.linspace(LOCATION2, LOCATION1, 2000)]).tolist()
diff --git a/selfdrive/test/test_onroad.py b/selfdrive/test/test_onroad.py
index 30632c7cd6..0e9d207da3 100755
--- a/selfdrive/test/test_onroad.py
+++ b/selfdrive/test/test_onroad.py
@@ -5,6 +5,7 @@ import json
 import os
 import pathlib
 import psutil
+import pytest
 import shutil
 import subprocess
 import time
@@ -98,6 +99,7 @@ def cputime_total(ct):
   return ct.cpuUser + ct.cpuSystem + ct.cpuChildrenUser + ct.cpuChildrenSystem
 
 
+@pytest.mark.tici
 class TestOnroad(unittest.TestCase):
 
   @classmethod
diff --git a/selfdrive/test/test_time_to_onroad.py b/selfdrive/test/test_time_to_onroad.py
index 501ce16b64..cc2cc6514e 100755
--- a/selfdrive/test/test_time_to_onroad.py
+++ b/selfdrive/test/test_time_to_onroad.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import os
+import pytest
 import time
 import subprocess
 
@@ -9,6 +10,7 @@ from openpilot.common.timeout import Timeout
 from openpilot.selfdrive.test.helpers import set_params_enabled
 
 
+@pytest.mark.tici
 def test_time_to_onroad():
   # launch
   set_params_enabled()
diff --git a/selfdrive/test/test_updated.py b/selfdrive/test/test_updated.py
index e679cd2c3b..dd79e03de4 100755
--- a/selfdrive/test/test_updated.py
+++ b/selfdrive/test/test_updated.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import datetime
 import os
+import pytest
 import time
 import tempfile
 import unittest
@@ -13,6 +14,7 @@ from openpilot.common.basedir import BASEDIR
 from openpilot.common.params import Params
 
 
+@pytest.mark.tici
 class TestUpdated(unittest.TestCase):
 
   def setUp(self):
diff --git a/system/camerad/test/test_camerad.py b/system/camerad/test/test_camerad.py
index b680c1df67..ad3a9fdc91 100755
--- a/system/camerad/test/test_camerad.py
+++ b/system/camerad/test/test_camerad.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import pytest
 import time
 import unittest
 import numpy as np
@@ -8,7 +9,6 @@ import cereal.messaging as messaging
 from cereal import log
 from cereal.services import SERVICE_LIST
 from openpilot.selfdrive.manager.process_config import managed_processes
-from openpilot.system.hardware import TICI
 
 TEST_TIMESPAN = 30
 LAG_FRAME_TOLERANCE = {log.FrameData.ImageSensor.ar0231: 0.5,  # ARs use synced pulses for frame starts
@@ -19,12 +19,10 @@ FRAME_DELTA_TOLERANCE = {log.FrameData.ImageSensor.ar0231: 1.0,
 CAMERAS = ('roadCameraState', 'driverCameraState', 'wideRoadCameraState')
 
 
+@pytest.mark.tici
 class TestCamerad(unittest.TestCase):
   @classmethod
   def setUpClass(cls):
-    if not TICI:
-      raise unittest.SkipTest
-
     # run camerad and record logs
     managed_processes['camerad'].start()
     time.sleep(3)
diff --git a/system/hardware/tici/tests/test_hardware.py b/system/hardware/tici/tests/test_hardware.py
index 7d377ac3fb..4abc86107b 100755
--- a/system/hardware/tici/tests/test_hardware.py
+++ b/system/hardware/tici/tests/test_hardware.py
@@ -1,20 +1,19 @@
 #!/usr/bin/env python3
+import pytest
 import time
 import unittest
 import numpy as np
 
-from openpilot.system.hardware import TICI
 from openpilot.system.hardware.tici.hardware import Tici
 
 HARDWARE = Tici()
 
+
+@pytest.mark.tici
 class TestHardware(unittest.TestCase):
 
   @classmethod
   def setUpClass(cls):
-    if not TICI:
-      raise unittest.SkipTest
-
     HARDWARE.initialize_hardware()
     HARDWARE.set_power_save(False)
 
diff --git a/system/hardware/tici/tests/test_power_draw.py b/system/hardware/tici/tests/test_power_draw.py
index 6411935749..c61d67e79d 100755
--- a/system/hardware/tici/tests/test_power_draw.py
+++ b/system/hardware/tici/tests/test_power_draw.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import pytest
 import unittest
 import time
 import threading
@@ -9,7 +10,7 @@ from typing import List
 
 import cereal.messaging as messaging
 from cereal.services import SERVICE_LIST
-from openpilot.system.hardware import HARDWARE, TICI
+from openpilot.system.hardware import HARDWARE
 from openpilot.system.hardware.tici.power_monitor import get_power
 from openpilot.selfdrive.manager.process_config import managed_processes
 from openpilot.selfdrive.manager.manager import manager_cleanup
@@ -44,13 +45,9 @@ def send_llk_msg(done):
     time.sleep(1/20.)
 
 
+@pytest.mark.tici
 class TestPowerDraw(unittest.TestCase):
 
-  @classmethod
-  def setUpClass(cls):
-    if not TICI:
-      raise unittest.SkipTest
-
   def setUp(self):
     HARDWARE.initialize_hardware()
     HARDWARE.set_power_save(False)
diff --git a/system/loggerd/tests/test_encoder.py b/system/loggerd/tests/test_encoder.py
index c46ac31b7f..bd076dc5f3 100755
--- a/system/loggerd/tests/test_encoder.py
+++ b/system/loggerd/tests/test_encoder.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import math
 import os
+import pytest
 import random
 import shutil
 import subprocess
@@ -31,14 +32,9 @@ CAMERAS = [
 FILE_SIZE_TOLERANCE = 0.5
 
 
+@pytest.mark.tici # TODO: all of loggerd should work on PC
 class TestEncoder(unittest.TestCase):
 
-  # TODO: all of loggerd should work on PC
-  @classmethod
-  def setUpClass(cls):
-    if not TICI:
-      raise unittest.SkipTest
-
   def setUp(self):
     self._clear_logs()
     os.environ["LOGGERD_TEST"] = "1"
diff --git a/system/sensord/rawgps/test_rawgps.py b/system/sensord/rawgps/test_rawgps.py
index 2a0ee656e2..3642438132 100755
--- a/system/sensord/rawgps/test_rawgps.py
+++ b/system/sensord/rawgps/test_rawgps.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import os
+import pytest
 import json
 import time
 import datetime
@@ -7,19 +8,16 @@ import unittest
 import subprocess
 
 import cereal.messaging as messaging
-from openpilot.system.hardware import TICI
 from openpilot.system.sensord.rawgps.rawgpsd import at_cmd, wait_for_modem
 from openpilot.selfdrive.manager.process_config import managed_processes
 
 GOOD_SIGNAL = bool(int(os.getenv("GOOD_SIGNAL", '0')))
 
 
+@pytest.mark.tici
 class TestRawgpsd(unittest.TestCase):
   @classmethod
   def setUpClass(cls):
-    if not TICI:
-      raise unittest.SkipTest
-
     os.system("sudo systemctl start systemd-resolved")
     os.system("sudo systemctl restart ModemManager lte")
     wait_for_modem()
diff --git a/system/sensord/tests/test_pigeond.py b/system/sensord/tests/test_pigeond.py
index b23afcb07c..f2ab43bbb7 100755
--- a/system/sensord/tests/test_pigeond.py
+++ b/system/sensord/tests/test_pigeond.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import pytest
 import time
 import unittest
 
@@ -7,16 +8,12 @@ from cereal.services import SERVICE_LIST
 from openpilot.common.gpio import gpio_read
 from openpilot.selfdrive.test.helpers import with_processes
 from openpilot.selfdrive.manager.process_config import managed_processes
-from openpilot.system.hardware import TICI
 from openpilot.system.hardware.tici.pins import GPIO
 
 
 # TODO: test TTFF when we have good A-GNSS
+@pytest.mark.tici
 class TestPigeond(unittest.TestCase):
-  @classmethod
-  def setUpClass(cls):
-    if not TICI:
-      raise unittest.SkipTest
 
   def tearDown(self):
     managed_processes['pigeond'].stop()
diff --git a/system/sensord/tests/test_sensord.py b/system/sensord/tests/test_sensord.py
index fba3ef79a4..0905248071 100755
--- a/system/sensord/tests/test_sensord.py
+++ b/system/sensord/tests/test_sensord.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import os
+import pytest
 import time
 import unittest
 import numpy as np
@@ -10,7 +11,6 @@ from cereal import log
 from cereal.services import SERVICE_LIST
 from openpilot.common.gpio import get_irqs_for_action
 from openpilot.common.timeout import Timeout
-from openpilot.system.hardware import TICI
 from openpilot.selfdrive.manager.process_config import managed_processes
 
 BMX = {
@@ -98,12 +98,10 @@ def read_sensor_events(duration_sec):
 
   return {k: v for k, v in events.items() if len(v) > 0}
 
+@pytest.mark.tici
 class TestSensord(unittest.TestCase):
   @classmethod
   def setUpClass(cls):
-    if not TICI:
-      raise unittest.SkipTest
-
     # enable LSM self test
     os.environ["LSM_SELF_TEST"] = "1"
 
diff --git a/tools/gpstest/test_gps.py b/tools/gpstest/test_gps.py
index 98f1ad84b8..bbd53ebfff 100755
--- a/tools/gpstest/test_gps.py
+++ b/tools/gpstest/test_gps.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import pytest
 import time
 import unittest
 import struct
@@ -6,7 +7,6 @@ import struct
 from openpilot.common.params import Params
 import cereal.messaging as messaging
 import openpilot.system.sensord.pigeond as pd
-from openpilot.system.hardware import TICI
 from openpilot.selfdrive.test.helpers import with_processes
 
 
@@ -107,12 +107,10 @@ def verify_time_to_first_fix(pigeon):
   assert ttff < 40, f"Time to first fix > 40s, {ttff}"
 
 
+@pytest.mark.tici
 class TestGPS(unittest.TestCase):
   @classmethod
   def setUpClass(cls):
-    if not TICI:
-      raise unittest.SkipTest
-
     ublox_available = Params().get_bool("UbloxAvailable")
     if not ublox_available:
       raise unittest.SkipTest
diff --git a/tools/gpstest/test_gps_qcom.py b/tools/gpstest/test_gps_qcom.py
index c399671715..2ea5556684 100755
--- a/tools/gpstest/test_gps_qcom.py
+++ b/tools/gpstest/test_gps_qcom.py
@@ -1,10 +1,10 @@
 #!/usr/bin/env python3
+import pytest
 import time
 import unittest
 import subprocess as sp
 
 from openpilot.common.params import Params
-from openpilot.system.hardware import TICI
 import cereal.messaging as messaging
 from openpilot.selfdrive.manager.process_config import managed_processes
 
@@ -30,12 +30,10 @@ def wait_for_location(socket, timeout):
     continue
 
 
+@pytest.mark.tici
 class TestGPS(unittest.TestCase):
   @classmethod
   def setUpClass(cls):
-    if not TICI:
-      raise unittest.SkipTest
-
     ublox_available = Params().get_bool("UbloxAvailable")
     if ublox_available:
       raise unittest.SkipTest