sadmen
/
openpilot_comma
mirror of https://github.com/commaai/openpilot.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ci: fix retry in test_camerad (#34167)

Browse Source 
* flaky was flaky

* delay
pull/34169/head
Maxime Desroches 5 months ago committed by GitHub
parent c7889a16be
commit 0902527e27
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 84 additions and 94 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      pyproject.toml
  2. 70
      system/camerad/test/test_camerad.py
  3. 107
      uv.lock

1
pyproject.toml
Unescape View File

@ -101,7 +101,6 @@ dev = [
"azure-identity",
"azure-storage-blob",
"dictdiffer",
"flaky",
"lru-dict",
"matplotlib",
"parameterized >=0.8, <0.9",

70
system/camerad/test/test_camerad.py
Unescape View File

@ -1,12 +1,12 @@
import pytest
import time
import numpy as np
from flaky import flaky
from collections import defaultdict
import cereal.messaging as messaging
from cereal import log
from cereal.services import SERVICE_LIST
from openpilot.common.retry import retry
from openpilot.system.manager.process_config import managed_processes
TEST_TIMESPAN = 10
@ -17,44 +17,46 @@ FRAME_DELTA_TOLERANCE = {log.FrameData.ImageSensor.ar0231: 1.0,
CAMERAS = ('roadCameraState', 'driverCameraState', 'wideRoadCameraState')
# TODO: this shouldn't be needed
@flaky(max_runs=3)
@pytest.mark.tici
class TestCamerad:
@classmethod
def setup_class(cls):
# run camerad and record logs
managed_processes['camerad'].start()
time.sleep(3)
socks = {c: messaging.sub_sock(c, conflate=False, timeout=100) for c in CAMERAS}
@retry(attempts=3, delay=5.0)
def setup_camerad(cls):
# run camerad and record logs
managed_processes['camerad'].start()
time.sleep(3)
socks = {c: messaging.sub_sock(c, conflate=False, timeout=100) for c in CAMERAS}
cls.logs = defaultdict(list)
start_time = time.monotonic()
while time.monotonic()- start_time < TEST_TIMESPAN:
for cam, s in socks.items():
cls.logs[cam] += messaging.drain_sock(s)
time.sleep(0.2)
managed_processes['camerad'].stop()
cls.logs = defaultdict(list)
start_time = time.monotonic()
while time.monotonic()- start_time < TEST_TIMESPAN:
for cam, s in socks.items():
cls.logs[cam] += messaging.drain_sock(s)
time.sleep(0.2)
managed_processes['camerad'].stop()
cls.log_by_frame_id = defaultdict(list)
cls.sensor_type = None
for cam, msgs in cls.logs.items():
if cls.sensor_type is None:
cls.sensor_type = getattr(msgs[0], msgs[0].which()).sensor.raw
expected_frames = SERVICE_LIST[cam].frequency * TEST_TIMESPAN
assert expected_frames*0.95 < len(msgs) < expected_frames*1.05, f"unexpected frame count {cam}: {expected_frames=}, got {len(msgs)}"
cls.log_by_frame_id = defaultdict(list)
cls.sensor_type = None
for cam, msgs in cls.logs.items():
if cls.sensor_type is None:
cls.sensor_type = getattr(msgs[0], msgs[0].which()).sensor.raw
expected_frames = SERVICE_LIST[cam].frequency * TEST_TIMESPAN
assert expected_frames*0.95 < len(msgs) < expected_frames*1.05, f"unexpected frame count {cam}: {expected_frames=}, got {len(msgs)}"
dts = np.abs(np.diff([getattr(m, m.which()).timestampSof/1e6 for m in msgs]) - 1000/SERVICE_LIST[cam].frequency)
assert (dts < FRAME_DELTA_TOLERANCE[cls.sensor_type]).all(), f"{cam} dts(ms) out of spec: max diff {dts.max()}, 99 percentile {np.percentile(dts, 99)}"
dts = np.abs(np.diff([getattr(m, m.which()).timestampSof/1e6 for m in msgs]) - 1000/SERVICE_LIST[cam].frequency)
assert (dts < FRAME_DELTA_TOLERANCE[cls.sensor_type]).all(), f"{cam} dts(ms) out of spec: max diff {dts.max()}, 99 percentile {np.percentile(dts, 99)}"
for m in msgs:
cls.log_by_frame_id[getattr(m, m.which()).frameId].append(m)
for m in msgs:
cls.log_by_frame_id[getattr(m, m.which()).frameId].append(m)
# strip beginning and end
for _ in range(3):
mn, mx = min(cls.log_by_frame_id.keys()), max(cls.log_by_frame_id.keys())
del cls.log_by_frame_id[mn]
del cls.log_by_frame_id[mx]
# strip beginning and end
for _ in range(3):
mn, mx = min(cls.log_by_frame_id.keys()), max(cls.log_by_frame_id.keys())
del cls.log_by_frame_id[mn]
del cls.log_by_frame_id[mx]
@pytest.mark.tici
class TestCamerad:
@classmethod
def setup_class(cls):
setup_camerad(cls)
def test_frame_skips(self):
skips = {}

107
uv.lock
Unescape View File

@ -338,7 +338,7 @@ name = "coloredlogs"
version = "15.0.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "humanfriendly" },
{ name = "humanfriendly", marker = "platform_system != 'Darwin'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/cc/c7/eed8f27100517e8c0e6b923d5f0845d0cb99763da6fdee00478f91db7325/coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0", size = 278520 }
wheels = [
@ -535,15 +535,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/b9/f8/feced7779d755758a52d1f6635d990b8d98dc0a29fa568bbe0625f18fdf3/filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0", size = 16163 },
]
[[package]]
name = "flaky"
version = "3.8.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/5b/c5/ef69119a01427204ff2db5fc8f98001087bcce719bbb94749dcd7b191365/flaky-3.8.1.tar.gz", hash = "sha256:47204a81ec905f3d5acfbd61daeabcada8f9d4031616d9bcb0618461729699f5", size = 25248 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/7f/b8/b830fc43663246c3f3dd1ae7dca4847b96ed992537e85311e27fa41ac40e/flaky-3.8.1-py2.py3-none-any.whl", hash = "sha256:194ccf4f0d3a22b2de7130f4b62e45e977ac1b5ccad74d4d48f3005dcc38815e", size = 19139 },
]
[[package]]
name = "flatbuffers"
version = "24.3.25"
@ -661,10 +652,10 @@ name = "gymnasium"
version = "1.0.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cloudpickle" },
{ name = "farama-notifications" },
{ name = "numpy" },
{ name = "typing-extensions" },
{ name = "cloudpickle", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "farama-notifications", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "numpy", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "typing-extensions", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/4e/12/1047b8fdbfcdce74022048d916e844ad7e6e1114d81d26a7aed657e3a76d/gymnasium-1.0.0.tar.gz", hash = "sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403", size = 821389 }
wheels = [
@ -676,7 +667,7 @@ name = "humanfriendly"
version = "10.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pyreadline3", marker = "sys_platform == 'win32'" },
{ name = "pyreadline3", marker = "platform_system != 'Darwin' and sys_platform == 'win32'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702 }
wheels = [
@ -971,22 +962,22 @@ name = "metadrive-simulator"
version = "0.4.2.3"
source = { url = "https://github.com/commaai/metadrive/releases/download/MetaDrive-minimal/metadrive_simulator-0.4.2.3-py3-none-any.whl" }
dependencies = [
{ name = "filelock" },
{ name = "gymnasium" },
{ name = "lxml" },
{ name = "matplotlib" },
{ name = "numpy" },
{ name = "opencv-python-headless" },
{ name = "panda3d" },
{ name = "panda3d-gltf" },
{ name = "pillow" },
{ name = "progressbar" },
{ name = "psutil" },
{ name = "pygments" },
{ name = "requests" },
{ name = "shapely" },
{ name = "tqdm" },
{ name = "yapf" },
{ name = "filelock", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "gymnasium", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "lxml", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "matplotlib", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "numpy", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "opencv-python-headless", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "panda3d", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "panda3d-gltf", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "pillow", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "progressbar", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "psutil", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "pygments", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "requests", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "shapely", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "tqdm", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "yapf", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
]
wheels = [
{ url = "https://github.com/commaai/metadrive/releases/download/MetaDrive-minimal/metadrive_simulator-0.4.2.3-py3-none-any.whl", hash = "sha256:6242d4e37e6c592d5eb1cadf497637540d3b754b89813a88c50a93c7fc88b02d" },
@ -1235,12 +1226,12 @@ name = "onnxruntime"
version = "1.20.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "coloredlogs" },
{ name = "flatbuffers" },
{ name = "numpy" },
{ name = "packaging" },
{ name = "protobuf" },
{ name = "sympy" },
{ name = "coloredlogs", marker = "platform_machine == 'aarch64' and platform_system == 'Linux'" },
{ name = "flatbuffers", marker = "platform_machine == 'aarch64' and platform_system == 'Linux'" },
{ name = "numpy", marker = "platform_machine == 'aarch64' and platform_system == 'Linux'" },
{ name = "packaging", marker = "platform_machine == 'aarch64' and platform_system == 'Linux'" },
{ name = "protobuf", marker = "platform_machine == 'aarch64' and platform_system == 'Linux'" },
{ name = "sympy", marker = "platform_machine == 'aarch64' and platform_system == 'Linux'" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/a5/da/c44bf9bd66cd6d9018a921f053f28d819445c4d84b4dd4777271b0fe52a2/onnxruntime-1.20.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6243e34d74423bdd1edf0ae9596dd61023b260f546ee17d701723915f06a9f7", size = 11955227 },
@ -1254,12 +1245,12 @@ name = "onnxruntime-gpu"
version = "1.20.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "coloredlogs" },
{ name = "flatbuffers" },
{ name = "numpy" },
{ name = "packaging" },
{ name = "protobuf" },
{ name = "sympy" },
{ name = "coloredlogs", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" },
{ name = "flatbuffers", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" },
{ name = "numpy", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" },
{ name = "packaging", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" },
{ name = "protobuf", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" },
{ name = "sympy", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/e0/a5/5c2287d61f359c7342e9d59d1e3dd728a982dea85f846c7af305a801c3ca/onnxruntime_gpu-1.20.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1795e8bc6f9a1488a4d51d242edc4232a5ae60ec44ab4d4b0a7c65b3d17fcbff", size = 291519550 },
@ -1271,7 +1262,7 @@ name = "opencv-python-headless"
version = "4.10.0.84"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "numpy" },
{ name = "numpy", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/2f/7e/d20f68a5f1487adf19d74378d349932a386b1ece3be9be9915e5986db468/opencv-python-headless-4.10.0.84.tar.gz", hash = "sha256:f2017c6101d7c2ef8d7bc3b414c37ff7f54d64413a1847d89970b6b7069b4e1a", size = 95117755 }
wheels = [
@ -1330,7 +1321,6 @@ dev = [
{ name = "azure-identity" },
{ name = "azure-storage-blob" },
{ name = "dictdiffer" },
{ name = "flaky" },
{ name = "lru-dict" },
{ name = "matplotlib" },
{ name = "parameterized" },
@ -1386,7 +1376,6 @@ requires-dist = [
{ name = "crcmod" },
{ name = "cython" },
{ name = "dictdiffer", marker = "extra == 'dev'" },
{ name = "flaky", marker = "extra == 'dev'" },
{ name = "future-fstrings" },
{ name = "hypothesis", marker = "extra == 'testing'", specifier = "==6.47.*" },
{ name = "inputs" },
@ -1480,8 +1469,8 @@ name = "panda3d-gltf"
version = "0.13"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "panda3d" },
{ name = "panda3d-simplepbr" },
{ name = "panda3d", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "panda3d-simplepbr", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/07/7f/9f18fc3fa843a080acb891af6bcc12262e7bdf1d194a530f7042bebfc81f/panda3d-gltf-0.13.tar.gz", hash = "sha256:d06d373bdd91cf530909b669f43080e599463bbf6d3ef00c3558bad6c6b19675", size = 25573 }
wheels = [
@ -1493,8 +1482,8 @@ name = "panda3d-simplepbr"
version = "0.12.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "panda3d" },
{ name = "typing-extensions" },
{ name = "panda3d", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "typing-extensions", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/b1/af/505608eef09d7f9b822e69dc7631cd14102650b8fe1b6f60d9562d2788d9/panda3d-simplepbr-0.12.0.tar.gz", hash = "sha256:c71d490afeeb3a90455dcfde1d30c41f321a38742a97d18834e5c31016331ed5", size = 1929980 }
wheels = [
@ -4365,9 +4354,9 @@ name = "pyopencl"
version = "2024.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "numpy" },
{ name = "platformdirs" },
{ name = "pytools" },
{ name = "numpy", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "platformdirs", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "pytools", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/ec/28/4679ea08b84532a67fd2d270c8f87aec64dab9ab99e618927b6a26ea063e/pyopencl-2024.3.tar.gz", hash = "sha256:d5d08de9b0a6d85695caba1769aceae4e7661f06951c507bd1ce8fb7a89e2413", size = 422604 }
wheels = [
@ -4421,7 +4410,7 @@ name = "pyqt5"
version = "5.15.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pyqt5-sip" },
{ name = "pyqt5-sip", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/28/6c/640e3f5c734c296a7193079a86842a789edb7988dca39eab44579088a1d1/PyQt5-5.15.2.tar.gz", hash = "sha256:372b08dc9321d1201e4690182697c5e7ffb2e0770e6b4a45519025134b12e4fc", size = 3265445 }
wheels = [
@ -4639,9 +4628,9 @@ name = "pytools"
version = "2024.1.10"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "platformdirs" },
{ name = "siphash24" },
{ name = "typing-extensions" },
{ name = "platformdirs", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "siphash24", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "typing-extensions", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/ee/0f/56e109c0307f831b5d598ad73976aaaa84b4d0e98da29a642e797eaa940c/pytools-2024.1.10.tar.gz", hash = "sha256:9af6f4b045212c49be32bb31fe19606c478ee4b09631886d05a32459f4ce0a12", size = 81741 }
wheels = [
@ -4944,7 +4933,7 @@ name = "shapely"
version = "2.0.6"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "numpy" },
{ name = "numpy", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/4a/89/0d20bac88016be35ff7d3c0c2ae64b477908f1b1dfa540c5d69ac7af07fe/shapely-2.0.6.tar.gz", hash = "sha256:997f6159b1484059ec239cacaa53467fd8b5564dabe186cd84ac2944663b0bf6", size = 282361 }
wheels = [
@ -5169,7 +5158,7 @@ name = "yapf"
version = "0.43.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "platformdirs" },
{ name = "platformdirs", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/23/97/b6f296d1e9cc1ec25c7604178b48532fa5901f721bcf1b8d8148b13e5588/yapf-0.43.0.tar.gz", hash = "sha256:00d3aa24bfedff9420b2e0d5d9f5ab6d9d4268e72afbf59bb3fa542781d5218e", size = 254907 }
wheels = [

Write Preview
Loading…
Cancel
Save