Restructure msgq (#32652)
* Update ref * Compiles * compiles * Refactor rest of libs * import all * small fiex * cleanup import * Need msgq simlink too * Add to openpilot docker too * try repo * Updates * Fix lint * fix docs * Try blank slate * Revert "Try blank slate" This reverts commit f078ce04acacfe115c19e23e86038b01e2b84a6d. * Maybe scons needs this to clear cache * fix tests * Disable test for now * Update SConstruct Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com> * Fix whitespace * Write skip normal * small fixes * add test path * Revert repo * linting * whitespace * Bump msgq --------- Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>pull/32657/head
Harald Schäfer
11 months ago
committed by
GitHub
parent
b573a4cc48
commit
e70dc90a45
31 changed files with 103 additions and 496 deletions
@ -1,193 +0,0 @@ |
||||
import os |
||||
import unittest |
||||
import multiprocessing |
||||
import platform |
||||
from parameterized import parameterized_class |
||||
from typing import Optional |
||||
|
||||
import cereal.messaging as messaging |
||||
|
||||
WAIT_TIMEOUT = 5 |
||||
|
||||
|
||||
@unittest.skipIf(platform.system() == "Darwin", "Events not supported on macOS") |
||||
class TestEvents(unittest.TestCase): |
||||
|
||||
def test_mutation(self): |
||||
handle = messaging.fake_event_handle("carState") |
||||
event = handle.recv_called_event |
||||
|
||||
self.assertFalse(event.peek()) |
||||
event.set() |
||||
self.assertTrue(event.peek()) |
||||
event.clear() |
||||
self.assertFalse(event.peek()) |
||||
|
||||
del event |
||||
|
||||
def test_wait(self): |
||||
handle = messaging.fake_event_handle("carState") |
||||
event = handle.recv_called_event |
||||
|
||||
event.set() |
||||
try: |
||||
event.wait(WAIT_TIMEOUT) |
||||
self.assertTrue(event.peek()) |
||||
except RuntimeError: |
||||
self.fail("event.wait() timed out") |
||||
|
||||
def test_wait_multiprocess(self): |
||||
handle = messaging.fake_event_handle("carState") |
||||
event = handle.recv_called_event |
||||
|
||||
def set_event_run(): |
||||
event.set() |
||||
|
||||
try: |
||||
p = multiprocessing.Process(target=set_event_run) |
||||
p.start() |
||||
event.wait(WAIT_TIMEOUT) |
||||
self.assertTrue(event.peek()) |
||||
except RuntimeError: |
||||
self.fail("event.wait() timed out") |
||||
|
||||
p.kill() |
||||
|
||||
def test_wait_zero_timeout(self): |
||||
handle = messaging.fake_event_handle("carState") |
||||
event = handle.recv_called_event |
||||
|
||||
try: |
||||
event.wait(0) |
||||
self.fail("event.wait() did not time out") |
||||
except RuntimeError: |
||||
self.assertFalse(event.peek()) |
||||
|
||||
|
||||
@unittest.skipIf(platform.system() == "Darwin", "FakeSockets not supported on macOS") |
||||
@unittest.skipIf("ZMQ" in os.environ, "FakeSockets not supported on ZMQ") |
||||
@parameterized_class([{"prefix": None}, {"prefix": "test"}]) |
||||
class TestFakeSockets(unittest.TestCase): |
||||
prefix: Optional[str] = None |
||||
|
||||
def setUp(self): |
||||
messaging.toggle_fake_events(True) |
||||
if self.prefix is not None: |
||||
messaging.set_fake_prefix(self.prefix) |
||||
else: |
||||
messaging.delete_fake_prefix() |
||||
|
||||
def tearDown(self): |
||||
messaging.toggle_fake_events(False) |
||||
messaging.delete_fake_prefix() |
||||
|
||||
def test_event_handle_init(self): |
||||
handle = messaging.fake_event_handle("controlsState", override=True) |
||||
|
||||
self.assertFalse(handle.enabled) |
||||
self.assertGreaterEqual(handle.recv_called_event.fd, 0) |
||||
self.assertGreaterEqual(handle.recv_ready_event.fd, 0) |
||||
|
||||
def test_non_managed_socket_state(self): |
||||
# non managed socket should have zero state |
||||
_ = messaging.pub_sock("ubloxGnss") |
||||
|
||||
handle = messaging.fake_event_handle("ubloxGnss", override=False) |
||||
|
||||
self.assertFalse(handle.enabled) |
||||
self.assertEqual(handle.recv_called_event.fd, 0) |
||||
self.assertEqual(handle.recv_ready_event.fd, 0) |
||||
|
||||
def test_managed_socket_state(self): |
||||
# managed socket should not change anything about the state |
||||
handle = messaging.fake_event_handle("ubloxGnss") |
||||
handle.enabled = True |
||||
|
||||
expected_enabled = handle.enabled |
||||
expected_recv_called_fd = handle.recv_called_event.fd |
||||
expected_recv_ready_fd = handle.recv_ready_event.fd |
||||
|
||||
_ = messaging.pub_sock("ubloxGnss") |
||||
|
||||
self.assertEqual(handle.enabled, expected_enabled) |
||||
self.assertEqual(handle.recv_called_event.fd, expected_recv_called_fd) |
||||
self.assertEqual(handle.recv_ready_event.fd, expected_recv_ready_fd) |
||||
|
||||
def test_sockets_enable_disable(self): |
||||
carState_handle = messaging.fake_event_handle("ubloxGnss", enable=True) |
||||
recv_called = carState_handle.recv_called_event |
||||
recv_ready = carState_handle.recv_ready_event |
||||
|
||||
pub_sock = messaging.pub_sock("ubloxGnss") |
||||
sub_sock = messaging.sub_sock("ubloxGnss") |
||||
|
||||
try: |
||||
carState_handle.enabled = True |
||||
recv_ready.set() |
||||
pub_sock.send(b"test") |
||||
_ = sub_sock.receive() |
||||
self.assertTrue(recv_called.peek()) |
||||
recv_called.clear() |
||||
|
||||
carState_handle.enabled = False |
||||
recv_ready.set() |
||||
pub_sock.send(b"test") |
||||
_ = sub_sock.receive() |
||||
self.assertFalse(recv_called.peek()) |
||||
except RuntimeError: |
||||
self.fail("event.wait() timed out") |
||||
|
||||
def test_synced_pub_sub(self): |
||||
def daemon_repub_process_run(): |
||||
pub_sock = messaging.pub_sock("ubloxGnss") |
||||
sub_sock = messaging.sub_sock("carState") |
||||
|
||||
frame = -1 |
||||
while True: |
||||
frame += 1 |
||||
msg = sub_sock.receive(non_blocking=True) |
||||
if msg is None: |
||||
print("none received") |
||||
continue |
||||
|
||||
bts = frame.to_bytes(8, 'little') |
||||
pub_sock.send(bts) |
||||
|
||||
carState_handle = messaging.fake_event_handle("carState", enable=True) |
||||
recv_called = carState_handle.recv_called_event |
||||
recv_ready = carState_handle.recv_ready_event |
||||
|
||||
p = multiprocessing.Process(target=daemon_repub_process_run) |
||||
p.start() |
||||
|
||||
pub_sock = messaging.pub_sock("carState") |
||||
sub_sock = messaging.sub_sock("ubloxGnss") |
||||
|
||||
try: |
||||
for i in range(10): |
||||
recv_called.wait(WAIT_TIMEOUT) |
||||
recv_called.clear() |
||||
|
||||
if i == 0: |
||||
sub_sock.receive(non_blocking=True) |
||||
|
||||
bts = i.to_bytes(8, 'little') |
||||
pub_sock.send(bts) |
||||
|
||||
recv_ready.set() |
||||
recv_called.wait(WAIT_TIMEOUT) |
||||
|
||||
msg = sub_sock.receive(non_blocking=True) |
||||
self.assertIsNotNone(msg) |
||||
self.assertEqual(len(msg), 8) |
||||
|
||||
frame = int.from_bytes(msg, 'little') |
||||
self.assertEqual(frame, i) |
||||
except RuntimeError: |
||||
self.fail("event.wait() timed out") |
||||
finally: |
||||
p.kill() |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
unittest.main() |
||||
@ -1,142 +0,0 @@ |
||||
import unittest |
||||
import time |
||||
import cereal.messaging as messaging |
||||
|
||||
import concurrent.futures |
||||
|
||||
|
||||
def poller(): |
||||
context = messaging.Context() |
||||
|
||||
p = messaging.Poller() |
||||
|
||||
sub = messaging.SubSocket() |
||||
sub.connect(context, 'controlsState') |
||||
p.registerSocket(sub) |
||||
|
||||
socks = p.poll(10000) |
||||
r = [s.receive(non_blocking=True) for s in socks] |
||||
|
||||
return r |
||||
|
||||
|
||||
class TestPoller(unittest.TestCase): |
||||
def test_poll_once(self): |
||||
context = messaging.Context() |
||||
|
||||
pub = messaging.PubSocket() |
||||
pub.connect(context, 'controlsState') |
||||
|
||||
with concurrent.futures.ThreadPoolExecutor() as e: |
||||
poll = e.submit(poller) |
||||
|
||||
time.sleep(0.1) # Slow joiner syndrome |
||||
|
||||
# Send message |
||||
pub.send(b"a") |
||||
|
||||
# Wait for poll result |
||||
result = poll.result() |
||||
|
||||
del pub |
||||
context.term() |
||||
|
||||
self.assertEqual(result, [b"a"]) |
||||
|
||||
def test_poll_and_create_many_subscribers(self): |
||||
context = messaging.Context() |
||||
|
||||
pub = messaging.PubSocket() |
||||
pub.connect(context, 'controlsState') |
||||
|
||||
with concurrent.futures.ThreadPoolExecutor() as e: |
||||
poll = e.submit(poller) |
||||
|
||||
time.sleep(0.1) # Slow joiner syndrome |
||||
c = messaging.Context() |
||||
for _ in range(10): |
||||
messaging.SubSocket().connect(c, 'controlsState') |
||||
|
||||
time.sleep(0.1) |
||||
|
||||
# Send message |
||||
pub.send(b"a") |
||||
|
||||
# Wait for poll result |
||||
result = poll.result() |
||||
|
||||
del pub |
||||
context.term() |
||||
|
||||
self.assertEqual(result, [b"a"]) |
||||
|
||||
def test_multiple_publishers_exception(self): |
||||
context = messaging.Context() |
||||
|
||||
with self.assertRaises(messaging.MultiplePublishersError): |
||||
pub1 = messaging.PubSocket() |
||||
pub1.connect(context, 'controlsState') |
||||
|
||||
pub2 = messaging.PubSocket() |
||||
pub2.connect(context, 'controlsState') |
||||
|
||||
pub1.send(b"a") |
||||
|
||||
del pub1 |
||||
del pub2 |
||||
context.term() |
||||
|
||||
def test_multiple_messages(self): |
||||
context = messaging.Context() |
||||
|
||||
pub = messaging.PubSocket() |
||||
pub.connect(context, 'controlsState') |
||||
|
||||
sub = messaging.SubSocket() |
||||
sub.connect(context, 'controlsState') |
||||
|
||||
time.sleep(0.1) # Slow joiner |
||||
|
||||
for i in range(1, 100): |
||||
pub.send(b'a'*i) |
||||
|
||||
msg_seen = False |
||||
i = 1 |
||||
while True: |
||||
r = sub.receive(non_blocking=True) |
||||
|
||||
if r is not None: |
||||
self.assertEqual(b'a'*i, r) |
||||
|
||||
msg_seen = True |
||||
i += 1 |
||||
|
||||
if r is None and msg_seen: # ZMQ sometimes receives nothing on the first receive |
||||
break |
||||
|
||||
del pub |
||||
del sub |
||||
context.term() |
||||
|
||||
def test_conflate(self): |
||||
context = messaging.Context() |
||||
|
||||
pub = messaging.PubSocket() |
||||
pub.connect(context, 'controlsState') |
||||
|
||||
sub = messaging.SubSocket() |
||||
sub.connect(context, 'controlsState', conflate=True) |
||||
|
||||
time.sleep(0.1) # Slow joiner |
||||
pub.send(b'a') |
||||
pub.send(b'b') |
||||
|
||||
self.assertEqual(b'b', sub.receive()) |
||||
|
||||
del pub |
||||
del sub |
||||
context.term() |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
unittest.main() |
||||
@ -1 +0,0 @@ |
||||
Subproject commit 615aea9b5519d2a3631fce4753bed29287fc4f9b |
||||
@ -0,0 +1 @@ |
||||
Subproject commit 381fc3d9dfe7d2ff40a075ff8c1f980ae2a62d19 |
||||
@ -1,3 +1,3 @@ |
||||
Import('env', 'cereal', 'messaging', 'common', 'socketmaster') |
||||
Import('env', 'messaging', 'common') |
||||
|
||||
env.Program('logcatd', 'logcatd_systemd.cc', LIBS=[cereal, socketmaster, messaging, common, 'zmq', 'capnp', 'kj', 'systemd', 'json11']) |
||||
env.Program('logcatd', 'logcatd_systemd.cc', LIBS=[messaging, common, 'systemd', 'json11']) |
||||
|
||||
@ -0,0 +1,39 @@ |
||||
FROM ghcr.io/commaai/openpilot-base:latest |
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \ |
||||
tmux \ |
||||
vim \ |
||||
&& rm -rf /var/lib/apt/lists/* |
||||
|
||||
# get same tmux config used on NEOS for debugging |
||||
RUN cd $HOME && \ |
||||
curl -O https://raw.githubusercontent.com/commaai/eon-neos-builder/master/devices/eon/home/.tmux.conf |
||||
|
||||
ENV OPENPILOT_PATH /tmp/openpilot |
||||
ENV PYTHONPATH ${OPENPILOT_PATH}:${PYTHONPATH} |
||||
|
||||
RUN mkdir -p ${OPENPILOT_PATH} |
||||
WORKDIR ${OPENPILOT_PATH} |
||||
|
||||
COPY SConstruct ${OPENPILOT_PATH} |
||||
|
||||
COPY ./openpilot ${OPENPILOT_PATH}/openpilot |
||||
COPY ./body ${OPENPILOT_PATH}/body |
||||
COPY ./third_party ${OPENPILOT_PATH}/third_party |
||||
COPY ./site_scons ${OPENPILOT_PATH}/site_scons |
||||
COPY ./rednose ${OPENPILOT_PATH}/rednose |
||||
COPY ./rednose_repo/site_scons ${OPENPILOT_PATH}/rednose_repo/site_scons |
||||
COPY ./common ${OPENPILOT_PATH}/common |
||||
COPY ./opendbc ${OPENPILOT_PATH}/opendbc |
||||
COPY ./cereal ${OPENPILOT_PATH}/cereal |
||||
COPY ./msgq_repo ${OPENPILOT_PATH}/msgq_repo |
||||
COPY ./msgq ${OPENPILOT_PATH}/msgq |
||||
COPY ./panda ${OPENPILOT_PATH}/panda |
||||
COPY ./selfdrive ${OPENPILOT_PATH}/selfdrive |
||||
COPY ./system ${OPENPILOT_PATH}/system |
||||
COPY ./tools ${OPENPILOT_PATH}/tools |
||||
COPY ./release ${OPENPILOT_PATH}/release |
||||
|
||||
RUN --mount=type=bind,source=.ci_cache/scons_cache,target=/tmp/scons_cache,rw scons -j$(nproc) --cache-readonly |
||||
|
||||
RUN python -c "from openpilot.selfdrive.test.helpers import set_params_enabled; set_params_enabled()" |
||||
Loading…
Reference in new issue