diff --git a/pyproject.toml b/pyproject.toml index 2bfcc19650..41b0e3aca5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,7 +101,6 @@ dev = [ "azure-identity", "azure-storage-blob", "dictdiffer", - "flaky", "lru-dict", "matplotlib", "parameterized >=0.8, <0.9", diff --git a/system/camerad/test/test_camerad.py b/system/camerad/test/test_camerad.py index 07a68e0020..c259fe788d 100644 --- a/system/camerad/test/test_camerad.py +++ b/system/camerad/test/test_camerad.py @@ -1,12 +1,12 @@ import pytest import time import numpy as np -from flaky import flaky from collections import defaultdict import cereal.messaging as messaging from cereal import log from cereal.services import SERVICE_LIST +from openpilot.common.retry import retry from openpilot.system.manager.process_config import managed_processes TEST_TIMESPAN = 10 @@ -17,44 +17,46 @@ FRAME_DELTA_TOLERANCE = {log.FrameData.ImageSensor.ar0231: 1.0, CAMERAS = ('roadCameraState', 'driverCameraState', 'wideRoadCameraState') -# TODO: this shouldn't be needed -@flaky(max_runs=3) -@pytest.mark.tici -class TestCamerad: - @classmethod - def setup_class(cls): - # run camerad and record logs - managed_processes['camerad'].start() - time.sleep(3) - socks = {c: messaging.sub_sock(c, conflate=False, timeout=100) for c in CAMERAS} +@retry(attempts=3, delay=5.0) +def setup_camerad(cls): + # run camerad and record logs + managed_processes['camerad'].start() + time.sleep(3) + socks = {c: messaging.sub_sock(c, conflate=False, timeout=100) for c in CAMERAS} + + cls.logs = defaultdict(list) + start_time = time.monotonic() + while time.monotonic()- start_time < TEST_TIMESPAN: + for cam, s in socks.items(): + cls.logs[cam] += messaging.drain_sock(s) + time.sleep(0.2) + managed_processes['camerad'].stop() - cls.logs = defaultdict(list) - start_time = time.monotonic() - while time.monotonic()- start_time < TEST_TIMESPAN: - for cam, s in socks.items(): - cls.logs[cam] += messaging.drain_sock(s) - time.sleep(0.2) - managed_processes['camerad'].stop() + cls.log_by_frame_id = defaultdict(list) + cls.sensor_type = None + for cam, msgs in cls.logs.items(): + if cls.sensor_type is None: + cls.sensor_type = getattr(msgs[0], msgs[0].which()).sensor.raw + expected_frames = SERVICE_LIST[cam].frequency * TEST_TIMESPAN + assert expected_frames*0.95 < len(msgs) < expected_frames*1.05, f"unexpected frame count {cam}: {expected_frames=}, got {len(msgs)}" - cls.log_by_frame_id = defaultdict(list) - cls.sensor_type = None - for cam, msgs in cls.logs.items(): - if cls.sensor_type is None: - cls.sensor_type = getattr(msgs[0], msgs[0].which()).sensor.raw - expected_frames = SERVICE_LIST[cam].frequency * TEST_TIMESPAN - assert expected_frames*0.95 < len(msgs) < expected_frames*1.05, f"unexpected frame count {cam}: {expected_frames=}, got {len(msgs)}" + dts = np.abs(np.diff([getattr(m, m.which()).timestampSof/1e6 for m in msgs]) - 1000/SERVICE_LIST[cam].frequency) + assert (dts < FRAME_DELTA_TOLERANCE[cls.sensor_type]).all(), f"{cam} dts(ms) out of spec: max diff {dts.max()}, 99 percentile {np.percentile(dts, 99)}" - dts = np.abs(np.diff([getattr(m, m.which()).timestampSof/1e6 for m in msgs]) - 1000/SERVICE_LIST[cam].frequency) - assert (dts < FRAME_DELTA_TOLERANCE[cls.sensor_type]).all(), f"{cam} dts(ms) out of spec: max diff {dts.max()}, 99 percentile {np.percentile(dts, 99)}" + for m in msgs: + cls.log_by_frame_id[getattr(m, m.which()).frameId].append(m) - for m in msgs: - cls.log_by_frame_id[getattr(m, m.which()).frameId].append(m) + # strip beginning and end + for _ in range(3): + mn, mx = min(cls.log_by_frame_id.keys()), max(cls.log_by_frame_id.keys()) + del cls.log_by_frame_id[mn] + del cls.log_by_frame_id[mx] - # strip beginning and end - for _ in range(3): - mn, mx = min(cls.log_by_frame_id.keys()), max(cls.log_by_frame_id.keys()) - del cls.log_by_frame_id[mn] - del cls.log_by_frame_id[mx] +@pytest.mark.tici +class TestCamerad: + @classmethod + def setup_class(cls): + setup_camerad(cls) def test_frame_skips(self): skips = {} diff --git a/uv.lock b/uv.lock index e5508cb972..085fdf369e 100644 --- a/uv.lock +++ b/uv.lock @@ -338,7 +338,7 @@ name = "coloredlogs" version = "15.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "humanfriendly" }, + { name = "humanfriendly", marker = "platform_system != 'Darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/c7/eed8f27100517e8c0e6b923d5f0845d0cb99763da6fdee00478f91db7325/coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0", size = 278520 } wheels = [ @@ -535,15 +535,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b9/f8/feced7779d755758a52d1f6635d990b8d98dc0a29fa568bbe0625f18fdf3/filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0", size = 16163 }, ] -[[package]] -name = "flaky" -version = "3.8.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5b/c5/ef69119a01427204ff2db5fc8f98001087bcce719bbb94749dcd7b191365/flaky-3.8.1.tar.gz", hash = "sha256:47204a81ec905f3d5acfbd61daeabcada8f9d4031616d9bcb0618461729699f5", size = 25248 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/b8/b830fc43663246c3f3dd1ae7dca4847b96ed992537e85311e27fa41ac40e/flaky-3.8.1-py2.py3-none-any.whl", hash = "sha256:194ccf4f0d3a22b2de7130f4b62e45e977ac1b5ccad74d4d48f3005dcc38815e", size = 19139 }, -] - [[package]] name = "flatbuffers" version = "24.3.25" @@ -661,10 +652,10 @@ name = "gymnasium" version = "1.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cloudpickle" }, - { name = "farama-notifications" }, - { name = "numpy" }, - { name = "typing-extensions" }, + { name = "cloudpickle", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "farama-notifications", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "numpy", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "typing-extensions", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4e/12/1047b8fdbfcdce74022048d916e844ad7e6e1114d81d26a7aed657e3a76d/gymnasium-1.0.0.tar.gz", hash = "sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403", size = 821389 } wheels = [ @@ -676,7 +667,7 @@ name = "humanfriendly" version = "10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyreadline3", marker = "sys_platform == 'win32'" }, + { name = "pyreadline3", marker = "platform_system != 'Darwin' and sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702 } wheels = [ @@ -971,22 +962,22 @@ name = "metadrive-simulator" version = "0.4.2.3" source = { url = "https://github.com/commaai/metadrive/releases/download/MetaDrive-minimal/metadrive_simulator-0.4.2.3-py3-none-any.whl" } dependencies = [ - { name = "filelock" }, - { name = "gymnasium" }, - { name = "lxml" }, - { name = "matplotlib" }, - { name = "numpy" }, - { name = "opencv-python-headless" }, - { name = "panda3d" }, - { name = "panda3d-gltf" }, - { name = "pillow" }, - { name = "progressbar" }, - { name = "psutil" }, - { name = "pygments" }, - { name = "requests" }, - { name = "shapely" }, - { name = "tqdm" }, - { name = "yapf" }, + { name = "filelock", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "gymnasium", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "lxml", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "matplotlib", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "numpy", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "opencv-python-headless", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "panda3d", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "panda3d-gltf", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "pillow", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "progressbar", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "psutil", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "pygments", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "requests", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "shapely", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "tqdm", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "yapf", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, ] wheels = [ { url = "https://github.com/commaai/metadrive/releases/download/MetaDrive-minimal/metadrive_simulator-0.4.2.3-py3-none-any.whl", hash = "sha256:6242d4e37e6c592d5eb1cadf497637540d3b754b89813a88c50a93c7fc88b02d" }, @@ -1235,12 +1226,12 @@ name = "onnxruntime" version = "1.20.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "coloredlogs" }, - { name = "flatbuffers" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "sympy" }, + { name = "coloredlogs", marker = "platform_machine == 'aarch64' and platform_system == 'Linux'" }, + { name = "flatbuffers", marker = "platform_machine == 'aarch64' and platform_system == 'Linux'" }, + { name = "numpy", marker = "platform_machine == 'aarch64' and platform_system == 'Linux'" }, + { name = "packaging", marker = "platform_machine == 'aarch64' and platform_system == 'Linux'" }, + { name = "protobuf", marker = "platform_machine == 'aarch64' and platform_system == 'Linux'" }, + { name = "sympy", marker = "platform_machine == 'aarch64' and platform_system == 'Linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a5/da/c44bf9bd66cd6d9018a921f053f28d819445c4d84b4dd4777271b0fe52a2/onnxruntime-1.20.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6243e34d74423bdd1edf0ae9596dd61023b260f546ee17d701723915f06a9f7", size = 11955227 }, @@ -1254,12 +1245,12 @@ name = "onnxruntime-gpu" version = "1.20.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "coloredlogs" }, - { name = "flatbuffers" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "sympy" }, + { name = "coloredlogs", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" }, + { name = "flatbuffers", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" }, + { name = "numpy", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" }, + { name = "packaging", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" }, + { name = "protobuf", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" }, + { name = "sympy", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/e0/a5/5c2287d61f359c7342e9d59d1e3dd728a982dea85f846c7af305a801c3ca/onnxruntime_gpu-1.20.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1795e8bc6f9a1488a4d51d242edc4232a5ae60ec44ab4d4b0a7c65b3d17fcbff", size = 291519550 }, @@ -1271,7 +1262,7 @@ name = "opencv-python-headless" version = "4.10.0.84" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, + { name = "numpy", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2f/7e/d20f68a5f1487adf19d74378d349932a386b1ece3be9be9915e5986db468/opencv-python-headless-4.10.0.84.tar.gz", hash = "sha256:f2017c6101d7c2ef8d7bc3b414c37ff7f54d64413a1847d89970b6b7069b4e1a", size = 95117755 } wheels = [ @@ -1330,7 +1321,6 @@ dev = [ { name = "azure-identity" }, { name = "azure-storage-blob" }, { name = "dictdiffer" }, - { name = "flaky" }, { name = "lru-dict" }, { name = "matplotlib" }, { name = "parameterized" }, @@ -1386,7 +1376,6 @@ requires-dist = [ { name = "crcmod" }, { name = "cython" }, { name = "dictdiffer", marker = "extra == 'dev'" }, - { name = "flaky", marker = "extra == 'dev'" }, { name = "future-fstrings" }, { name = "hypothesis", marker = "extra == 'testing'", specifier = "==6.47.*" }, { name = "inputs" }, @@ -1480,8 +1469,8 @@ name = "panda3d-gltf" version = "0.13" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "panda3d" }, - { name = "panda3d-simplepbr" }, + { name = "panda3d", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "panda3d-simplepbr", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/07/7f/9f18fc3fa843a080acb891af6bcc12262e7bdf1d194a530f7042bebfc81f/panda3d-gltf-0.13.tar.gz", hash = "sha256:d06d373bdd91cf530909b669f43080e599463bbf6d3ef00c3558bad6c6b19675", size = 25573 } wheels = [ @@ -1493,8 +1482,8 @@ name = "panda3d-simplepbr" version = "0.12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "panda3d" }, - { name = "typing-extensions" }, + { name = "panda3d", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "typing-extensions", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b1/af/505608eef09d7f9b822e69dc7631cd14102650b8fe1b6f60d9562d2788d9/panda3d-simplepbr-0.12.0.tar.gz", hash = "sha256:c71d490afeeb3a90455dcfde1d30c41f321a38742a97d18834e5c31016331ed5", size = 1929980 } wheels = [ @@ -4365,9 +4354,9 @@ name = "pyopencl" version = "2024.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, - { name = "platformdirs" }, - { name = "pytools" }, + { name = "numpy", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "platformdirs", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "pytools", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ec/28/4679ea08b84532a67fd2d270c8f87aec64dab9ab99e618927b6a26ea063e/pyopencl-2024.3.tar.gz", hash = "sha256:d5d08de9b0a6d85695caba1769aceae4e7661f06951c507bd1ce8fb7a89e2413", size = 422604 } wheels = [ @@ -4421,7 +4410,7 @@ name = "pyqt5" version = "5.15.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyqt5-sip" }, + { name = "pyqt5-sip", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/28/6c/640e3f5c734c296a7193079a86842a789edb7988dca39eab44579088a1d1/PyQt5-5.15.2.tar.gz", hash = "sha256:372b08dc9321d1201e4690182697c5e7ffb2e0770e6b4a45519025134b12e4fc", size = 3265445 } wheels = [ @@ -4639,9 +4628,9 @@ name = "pytools" version = "2024.1.10" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "platformdirs" }, - { name = "siphash24" }, - { name = "typing-extensions" }, + { name = "platformdirs", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "siphash24", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, + { name = "typing-extensions", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ee/0f/56e109c0307f831b5d598ad73976aaaa84b4d0e98da29a642e797eaa940c/pytools-2024.1.10.tar.gz", hash = "sha256:9af6f4b045212c49be32bb31fe19606c478ee4b09631886d05a32459f4ce0a12", size = 81741 } wheels = [ @@ -4944,7 +4933,7 @@ name = "shapely" version = "2.0.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, + { name = "numpy", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4a/89/0d20bac88016be35ff7d3c0c2ae64b477908f1b1dfa540c5d69ac7af07fe/shapely-2.0.6.tar.gz", hash = "sha256:997f6159b1484059ec239cacaa53467fd8b5564dabe186cd84ac2944663b0bf6", size = 282361 } wheels = [ @@ -5169,7 +5158,7 @@ name = "yapf" version = "0.43.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "platformdirs" }, + { name = "platformdirs", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/23/97/b6f296d1e9cc1ec25c7604178b48532fa5901f721bcf1b8d8148b13e5588/yapf-0.43.0.tar.gz", hash = "sha256:00d3aa24bfedff9420b2e0d5d9f5ab6d9d4268e72afbf59bb3fa542781d5218e", size = 254907 } wheels = [