From 83fc734b8f6bbb75698fc575ddab1d63cb706067 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 2 Feb 2024 11:28:42 -0800 Subject: [PATCH 1/5] add neural ff weights to release --- release/files_common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/files_common b/release/files_common index 91945a0621..1158d2c552 100644 --- a/release/files_common +++ b/release/files_common @@ -95,7 +95,7 @@ selfdrive/car/ecu_addrs.py selfdrive/car/isotp_parallel_query.py selfdrive/car/tests/__init__.py selfdrive/car/tests/test_car_interfaces.py -selfdrive/car/torque_data/*.toml +selfdrive/car/torque_data/* selfdrive/car/body/*.py selfdrive/car/chrysler/*.py From 71236204bb78d64feb3d46680176dfb37741dc5d Mon Sep 17 00:00:00 2001 From: Hoang Bui <47828508+bongbui321@users.noreply.github.com> Date: Fri, 2 Feb 2024 15:06:05 -0500 Subject: [PATCH 2/5] Fix self.started value pass in metadrive test (#31153) * fix value pass * fix test --------- Co-authored-by: Justin Newberry --- tools/sim/bridge/common.py | 8 ++++---- tools/sim/tests/test_metadrive_bridge.py | 3 +-- tools/sim/tests/test_sim_bridge.py | 5 ++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/tools/sim/bridge/common.py b/tools/sim/bridge/common.py index bdd53852c0..91ab0b6f07 100644 --- a/tools/sim/bridge/common.py +++ b/tools/sim/bridge/common.py @@ -2,7 +2,7 @@ import signal import threading import functools -from multiprocessing import Process, Queue +from multiprocessing import Process, Queue, Value from abc import ABC, abstractmethod from typing import Optional @@ -39,7 +39,7 @@ class SimulatorBridge(ABC): self._exit_event = threading.Event() self._threads = [] self._keep_alive = True - self.started = False + self.started = Value('i', False) signal.signal(signal.SIGTERM, self._on_shutdown) self._exit = threading.Event() self.simulator_state = SimulatorState() @@ -61,7 +61,7 @@ class SimulatorBridge(ABC): self.close() def close(self): - self.started = False + self.started.value = False self._exit_event.set() if self.world is not None: @@ -181,6 +181,6 @@ Ignition: {self.simulator_state.ignition} Engaged: {self.simulator_state.is_enga if self.rk.frame % 25 == 0: self.print_status() - self.started = True + self.started.value = True self.rk.keep_time() diff --git a/tools/sim/tests/test_metadrive_bridge.py b/tools/sim/tests/test_metadrive_bridge.py index 4d784956d2..6cb8e1465e 100755 --- a/tools/sim/tests/test_metadrive_bridge.py +++ b/tools/sim/tests/test_metadrive_bridge.py @@ -2,14 +2,13 @@ import pytest import unittest -from openpilot.tools.sim.run_bridge import parse_args from openpilot.tools.sim.bridge.metadrive.metadrive_bridge import MetaDriveBridge from openpilot.tools.sim.tests.test_sim_bridge import TestSimBridgeBase @pytest.mark.slow class TestMetaDriveBridge(TestSimBridgeBase): def create_bridge(self): - return MetaDriveBridge(parse_args([])) + return MetaDriveBridge(False, False) if __name__ == "__main__": diff --git a/tools/sim/tests/test_sim_bridge.py b/tools/sim/tests/test_sim_bridge.py index 850fd83bac..504914c562 100644 --- a/tools/sim/tests/test_sim_bridge.py +++ b/tools/sim/tests/test_sim_bridge.py @@ -3,7 +3,7 @@ import subprocess import time import unittest -from multiprocessing import Queue, Value +from multiprocessing import Queue from cereal import messaging from openpilot.common.basedir import BASEDIR @@ -27,7 +27,6 @@ class TestSimBridgeBase(unittest.TestCase): sm = messaging.SubMaster(['controlsState', 'onroadEvents', 'managerState']) q = Queue() bridge = self.create_bridge() - bridge.started = Value('b', False) p_bridge = bridge.run(q, retries=10) self.processes.append(p_bridge) @@ -35,7 +34,7 @@ class TestSimBridgeBase(unittest.TestCase): # Wait for bridge to startup start_waiting = time.monotonic() - while not bridge.started and time.monotonic() < start_waiting + max_time_per_step: + while not bridge.started.value and time.monotonic() < start_waiting + max_time_per_step: time.sleep(0.1) self.assertEqual(p_bridge.exitcode, None, f"Bridge process should be running, but exited with code {p_bridge.exitcode}") From a5766e27967707d51c4e7181a3509c78b3671d16 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 2 Feb 2024 12:53:12 -0800 Subject: [PATCH 3/5] encoderd: get frame size over vipc (#31276) * encoderd: get frame size over vipc * bump cereal * fix ffmpeg * no print --- cereal | 2 +- system/loggerd/encoder/encoder.cc | 6 ++++++ system/loggerd/encoder/encoder.h | 3 ++- system/loggerd/encoder/ffmpeg_encoder.cc | 14 +++++++------- system/loggerd/encoder/v4l_encoder.cc | 4 ++-- system/loggerd/loggerd.cc | 2 +- system/loggerd/loggerd.h | 4 ++-- 7 files changed, 21 insertions(+), 14 deletions(-) diff --git a/cereal b/cereal index a6ade85c9d..c54369f8ad 160000 --- a/cereal +++ b/cereal @@ -1 +1 @@ -Subproject commit a6ade85c9dd6652fde547b9e089a297f67606dcf +Subproject commit c54369f8ad4e0bcb18c96feb4334755c6f65e8f1 diff --git a/system/loggerd/encoder/encoder.cc b/system/loggerd/encoder/encoder.cc index 0aba4b8b49..c4bd91bcf7 100644 --- a/system/loggerd/encoder/encoder.cc +++ b/system/loggerd/encoder/encoder.cc @@ -2,6 +2,10 @@ VideoEncoder::VideoEncoder(const EncoderInfo &encoder_info, int in_width, int in_height) : encoder_info(encoder_info), in_width(in_width), in_height(in_height) { + + out_width = encoder_info.frame_width > 0 ? encoder_info.frame_width : in_width; + out_height = encoder_info.frame_height > 0 ? encoder_info.frame_height : in_height; + pm.reset(new PubMaster({encoder_info.publish_name})); } @@ -25,6 +29,8 @@ void VideoEncoder::publisher_publish(VideoEncoder *e, int segment_num, uint32_t edata.setFlags(flags); edata.setLen(dat.size()); edat.setData(dat); + edat.setWidth(out_width); + edat.setHeight(out_height); if (flags & V4L2_BUF_FLAG_KEYFRAME) edat.setHeader(header); uint32_t bytes_size = capnp::computeSerializedSizeInWords(msg) * sizeof(capnp::word); diff --git a/system/loggerd/encoder/encoder.h b/system/loggerd/encoder/encoder.h index 9c23928cc6..7c203f9193 100644 --- a/system/loggerd/encoder/encoder.h +++ b/system/loggerd/encoder/encoder.h @@ -22,10 +22,11 @@ public: virtual void encoder_open(const char* path) = 0; virtual void encoder_close() = 0; - static void publisher_publish(VideoEncoder *e, int segment_num, uint32_t idx, VisionIpcBufExtra &extra, unsigned int flags, kj::ArrayPtr header, kj::ArrayPtr dat); + void publisher_publish(VideoEncoder *e, int segment_num, uint32_t idx, VisionIpcBufExtra &extra, unsigned int flags, kj::ArrayPtr header, kj::ArrayPtr dat); protected: int in_width, in_height; + int out_width, out_height; const EncoderInfo encoder_info; private: diff --git a/system/loggerd/encoder/ffmpeg_encoder.cc b/system/loggerd/encoder/ffmpeg_encoder.cc index f44f2fbed7..9d992f088d 100644 --- a/system/loggerd/encoder/ffmpeg_encoder.cc +++ b/system/loggerd/encoder/ffmpeg_encoder.cc @@ -29,16 +29,16 @@ FfmpegEncoder::FfmpegEncoder(const EncoderInfo &encoder_info, int in_width, int frame = av_frame_alloc(); assert(frame); frame->format = AV_PIX_FMT_YUV420P; - frame->width = encoder_info.frame_width; - frame->height = encoder_info.frame_height; - frame->linesize[0] = encoder_info.frame_width; - frame->linesize[1] = encoder_info.frame_width/2; - frame->linesize[2] = encoder_info.frame_width/2; + frame->width = out_width; + frame->height = out_height; + frame->linesize[0] = out_width; + frame->linesize[1] = out_width/2; + frame->linesize[2] = out_width/2; convert_buf.resize(in_width * in_height * 3 / 2); - if (in_width != encoder_info.frame_width || in_height != encoder_info.frame_height) { - downscale_buf.resize(encoder_info.frame_width * encoder_info.frame_height * 3 / 2); + if (in_width != out_width || in_height != out_height) { + downscale_buf.resize(out_width * out_height * 3 / 2); } } diff --git a/system/loggerd/encoder/v4l_encoder.cc b/system/loggerd/encoder/v4l_encoder.cc index 571f5979e2..2bd2863126 100644 --- a/system/loggerd/encoder/v4l_encoder.cc +++ b/system/loggerd/encoder/v4l_encoder.cc @@ -164,8 +164,8 @@ V4LEncoder::V4LEncoder(const EncoderInfo &encoder_info, int in_width, int in_hei .fmt = { .pix_mp = { // downscales are free with v4l - .width = (unsigned int)encoder_info.frame_width, - .height = (unsigned int)encoder_info.frame_height, + .width = (unsigned int)(out_width), + .height = (unsigned int)(out_height), .pixelformat = (encoder_info.encode_type == cereal::EncodeIndex::Type::FULL_H_E_V_C) ? V4L2_PIX_FMT_HEVC : V4L2_PIX_FMT_H264, .field = V4L2_FIELD_ANY, .colorspace = V4L2_COLORSPACE_DEFAULT, diff --git a/system/loggerd/loggerd.cc b/system/loggerd/loggerd.cc index 27dfa187c4..3c0ffc1667 100644 --- a/system/loggerd/loggerd.cc +++ b/system/loggerd/loggerd.cc @@ -116,7 +116,7 @@ int handle_encoder_msg(LoggerdState *s, Message *msg, std::string &name, struct assert(encoder_info.filename != NULL); re.writer.reset(new VideoWriter(s->logger.segmentPath().c_str(), encoder_info.filename, idx.getType() != cereal::EncodeIndex::Type::FULL_H_E_V_C, - encoder_info.frame_width, encoder_info.frame_height, encoder_info.fps, idx.getType())); + edata.getWidth(), edata.getHeight(), encoder_info.fps, idx.getType())); // write the header auto header = edata.getHeader(); re.writer->write((uint8_t *)header.begin(), header.size(), idx.getTimestampEof()/1000, true, false); diff --git a/system/loggerd/loggerd.h b/system/loggerd/loggerd.h index cfc06c28d3..ea288f4861 100644 --- a/system/loggerd/loggerd.h +++ b/system/loggerd/loggerd.h @@ -35,8 +35,8 @@ public: const char *publish_name; const char *filename = NULL; bool record = true; - int frame_width = 1928; - int frame_height = 1208; + int frame_width = -1; + int frame_height = -1; int fps = MAIN_FPS; int bitrate = MAIN_BITRATE; cereal::EncodeIndex::Type encode_type = Hardware::PC() ? cereal::EncodeIndex::Type::BIG_BOX_LOSSLESS From 0277fc5548bc87a17ae34b914df0593d034e5e94 Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Fri, 2 Feb 2024 17:18:01 -0500 Subject: [PATCH 4/5] Pytest: consistent hardware state for tici tests (#31279) * consistent hardware * consistent hardware * moved * this too * ruff * s * duplicated --------- Co-authored-by: Comma Device --- Jenkinsfile | 2 +- conftest.py | 18 +++++++++++++++--- selfdrive/test/test_onroad.py | 8 +------- system/hardware/tici/tests/test_hardware.py | 7 +------ system/hardware/tici/tests/test_power_draw.py | 5 +---- 5 files changed, 19 insertions(+), 21 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 816c2971c0..d3b3444991 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -208,7 +208,7 @@ node { deviceStage("tici", "tici-common", ["UNSAFE=1"], [ ["build", "cd selfdrive/manager && ./build.py"], ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py", ["panda/", "selfdrive/boardd/"]], - ["test power draw", "./system/hardware/tici/tests/test_power_draw.py"], + ["test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"], ["test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py"], ["test pigeond", "pytest system/sensord/tests/test_pigeond.py"], ["test manager", "pytest selfdrive/manager/test/test_manager.py"], diff --git a/conftest.py b/conftest.py index c4eb259a35..e215bb8b4c 100644 --- a/conftest.py +++ b/conftest.py @@ -4,7 +4,7 @@ import random from openpilot.common.prefix import OpenpilotPrefix from openpilot.selfdrive.manager import manager -from openpilot.system.hardware import TICI +from openpilot.system.hardware import TICI, HARDWARE def pytest_sessionstart(session): @@ -57,12 +57,24 @@ def openpilot_class_fixture(): os.environ.update(starting_env) +@pytest.fixture(scope="class") +def tici_setup_fixture(): + """Ensure a consistent state for tests on-device""" + HARDWARE.initialize_hardware() + HARDWARE.set_power_save(False) + os.system("pkill -9 -f athena") + os.system("rm /dev/shm/*") + + @pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems(config, items): skipper = pytest.mark.skip(reason="Skipping tici test on PC") for item in items: - if not TICI and "tici" in item.keywords: - item.add_marker(skipper) + if "tici" in item.keywords: + if not TICI: + item.add_marker(skipper) + else: + item.fixturenames.append('tici_setup_fixture') if "xdist_group_class_property" in item.keywords: class_property_name = item.get_closest_marker('xdist_group_class_property').args[0] diff --git a/selfdrive/test/test_onroad.py b/selfdrive/test/test_onroad.py index 335da73232..036eacfa48 100755 --- a/selfdrive/test/test_onroad.py +++ b/selfdrive/test/test_onroad.py @@ -112,17 +112,11 @@ class TestOnroad(unittest.TestCase): # setup env params = Params() - if "CI" in os.environ: - params.clear_all() params.remove("CurrentRoute") set_params_enabled() os.environ['TESTING_CLOSET'] = '1' if os.path.exists(Paths.log_root()): shutil.rmtree(Paths.log_root()) - os.system("rm /dev/shm/*") - - # Make sure athena isn't running - os.system("pkill -9 -f athena") # start manager and run openpilot for a minute proc = None @@ -429,4 +423,4 @@ class TestOnroad(unittest.TestCase): if __name__ == "__main__": - unittest.main() + pytest.main() diff --git a/system/hardware/tici/tests/test_hardware.py b/system/hardware/tici/tests/test_hardware.py index 4abc86107b..0c436595ee 100755 --- a/system/hardware/tici/tests/test_hardware.py +++ b/system/hardware/tici/tests/test_hardware.py @@ -12,11 +12,6 @@ HARDWARE = Tici() @pytest.mark.tici class TestHardware(unittest.TestCase): - @classmethod - def setUpClass(cls): - HARDWARE.initialize_hardware() - HARDWARE.set_power_save(False) - def test_power_save_time(self): ts = [] for _ in range(5): @@ -30,4 +25,4 @@ class TestHardware(unittest.TestCase): if __name__ == "__main__": - unittest.main() + pytest.main() diff --git a/system/hardware/tici/tests/test_power_draw.py b/system/hardware/tici/tests/test_power_draw.py index 0518eec543..db75f79af5 100755 --- a/system/hardware/tici/tests/test_power_draw.py +++ b/system/hardware/tici/tests/test_power_draw.py @@ -11,7 +11,6 @@ import cereal.messaging as messaging from cereal.services import SERVICE_LIST from openpilot.common.mock import mock_messages from openpilot.selfdrive.car.car_helpers import write_car_param -from openpilot.system.hardware import HARDWARE from openpilot.system.hardware.tici.power_monitor import get_power from openpilot.selfdrive.manager.process_config import managed_processes from openpilot.selfdrive.manager.manager import manager_cleanup @@ -41,8 +40,6 @@ PROCS = [ class TestPowerDraw(unittest.TestCase): def setUp(self): - HARDWARE.initialize_hardware() - HARDWARE.set_power_save(False) write_car_param() # wait a bit for power save to disable @@ -88,4 +85,4 @@ class TestPowerDraw(unittest.TestCase): if __name__ == "__main__": - unittest.main() + pytest.main() From 28a15dbfe916c40d30a4901484e2d75fcb8af4cc Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Fri, 2 Feb 2024 18:45:59 -0500 Subject: [PATCH 5/5] CI: enable logical cpus for all selfdrive tests (#31281) logical --- .github/workflows/selfdrive_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 964c36776e..860cfa48f7 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -26,7 +26,7 @@ env: RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/bash -c - PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 + PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 -n logical jobs: build_release: @@ -182,7 +182,7 @@ jobs: run: | ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \ export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && \ - $PYTEST --timeout 60 -m 'not slow' -n $(nproc) && \ + $PYTEST --timeout 60 -m 'not slow' && \ ./selfdrive/ui/tests/create_test_translations.sh && \ QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \ ./selfdrive/ui/tests/test_translations.py"