Merge remote-tracking branch 'upstream/master' into car-interfaces-test-more

.github/workflows/selfdrive_tests.yaml

@@ -26,7 +26,7 @@ env:
 
   RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/bash -c
 
-  PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0
+  PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 -n logical
 
 jobs:
   build_release:
@@ -182,7 +182,7 @@ jobs:
       run: |
         ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
                         export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && \
-                        $PYTEST --timeout 60 -m 'not slow' -n $(nproc) && \
+                        $PYTEST --timeout 60 -m 'not slow' && \
                         ./selfdrive/ui/tests/create_test_translations.sh && \
                         QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
                         ./selfdrive/ui/tests/test_translations.py"

Jenkinsfile

@@ -208,7 +208,7 @@ node {
         deviceStage("tici", "tici-common", ["UNSAFE=1"], [
           ["build", "cd selfdrive/manager && ./build.py"],
           ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py", ["panda/", "selfdrive/boardd/"]],
-          ["test power draw", "./system/hardware/tici/tests/test_power_draw.py"],
+          ["test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"],
           ["test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py"],
           ["test pigeond", "pytest system/sensord/tests/test_pigeond.py"],
           ["test manager", "pytest selfdrive/manager/test/test_manager.py"],

cereal

@@ -1 +1 @@
-Subproject commit a6ade85c9dd6652fde547b9e089a297f67606dcf
+Subproject commit c54369f8ad4e0bcb18c96feb4334755c6f65e8f1

conftest.py

@@ -4,7 +4,7 @@ import random
 
 from openpilot.common.prefix import OpenpilotPrefix
 from openpilot.selfdrive.manager import manager
-from openpilot.system.hardware import TICI
+from openpilot.system.hardware import TICI, HARDWARE
 
 
 def pytest_sessionstart(session):
@@ -57,12 +57,24 @@ def openpilot_class_fixture():
   os.environ.update(starting_env)
 
 
+@pytest.fixture(scope="class")
+def tici_setup_fixture():
+  """Ensure a consistent state for tests on-device"""
+  HARDWARE.initialize_hardware()
+  HARDWARE.set_power_save(False)
+  os.system("pkill -9 -f athena")
+  os.system("rm /dev/shm/*")
+
+
 @pytest.hookimpl(tryfirst=True)
 def pytest_collection_modifyitems(config, items):
   skipper = pytest.mark.skip(reason="Skipping tici test on PC")
   for item in items:
-    if not TICI and "tici" in item.keywords:
+    if "tici" in item.keywords:
-      item.add_marker(skipper)
+      if not TICI:
+        item.add_marker(skipper)
+      else:
+        item.fixturenames.append('tici_setup_fixture')
 
     if "xdist_group_class_property" in item.keywords:
       class_property_name = item.get_closest_marker('xdist_group_class_property').args[0]

release/files_common

@@ -95,7 +95,7 @@ selfdrive/car/ecu_addrs.py
 selfdrive/car/isotp_parallel_query.py
 selfdrive/car/tests/__init__.py
 selfdrive/car/tests/test_car_interfaces.py
-selfdrive/car/torque_data/*.toml
+selfdrive/car/torque_data/*
 
 selfdrive/car/body/*.py
 selfdrive/car/chrysler/*.py

selfdrive/test/test_onroad.py

@@ -112,17 +112,11 @@ class TestOnroad(unittest.TestCase):
 
     # setup env
     params = Params()
-    if "CI" in os.environ:
-      params.clear_all()
     params.remove("CurrentRoute")
     set_params_enabled()
     os.environ['TESTING_CLOSET'] = '1'
     if os.path.exists(Paths.log_root()):
       shutil.rmtree(Paths.log_root())
-    os.system("rm /dev/shm/*")
-
-    # Make sure athena isn't running
-    os.system("pkill -9 -f athena")
 
     # start manager and run openpilot for a minute
     proc = None
@@ -429,4 +423,4 @@ class TestOnroad(unittest.TestCase):
 
 
 if __name__ == "__main__":
-  unittest.main()
+  pytest.main()

system/hardware/tici/tests/test_hardware.py

@@ -12,11 +12,6 @@ HARDWARE = Tici()
 @pytest.mark.tici
 class TestHardware(unittest.TestCase):
 
-  @classmethod
-  def setUpClass(cls):
-    HARDWARE.initialize_hardware()
-    HARDWARE.set_power_save(False)
-
   def test_power_save_time(self):
     ts = []
     for _ in range(5):
@@ -30,4 +25,4 @@ class TestHardware(unittest.TestCase):
 
 
 if __name__ == "__main__":
-  unittest.main()
+  pytest.main()

system/hardware/tici/tests/test_power_draw.py

@@ -11,7 +11,6 @@ import cereal.messaging as messaging
 from cereal.services import SERVICE_LIST
 from openpilot.common.mock import mock_messages
 from openpilot.selfdrive.car.car_helpers import write_car_param
-from openpilot.system.hardware import HARDWARE
 from openpilot.system.hardware.tici.power_monitor import get_power
 from openpilot.selfdrive.manager.process_config import managed_processes
 from openpilot.selfdrive.manager.manager import manager_cleanup
@@ -41,8 +40,6 @@ PROCS = [
 class TestPowerDraw(unittest.TestCase):
 
   def setUp(self):
-    HARDWARE.initialize_hardware()
-    HARDWARE.set_power_save(False)
     write_car_param()
 
     # wait a bit for power save to disable
@@ -88,4 +85,4 @@ class TestPowerDraw(unittest.TestCase):
 
 
 if __name__ == "__main__":
-  unittest.main()
+  pytest.main()

system/loggerd/encoder/encoder.cc

@@ -2,6 +2,10 @@
 
 VideoEncoder::VideoEncoder(const EncoderInfo &encoder_info, int in_width, int in_height)
     : encoder_info(encoder_info), in_width(in_width), in_height(in_height) {
+
+  out_width = encoder_info.frame_width > 0 ? encoder_info.frame_width : in_width;
+  out_height = encoder_info.frame_height > 0 ? encoder_info.frame_height : in_height;
+
   pm.reset(new PubMaster({encoder_info.publish_name}));
 }
 
@@ -25,6 +29,8 @@ void VideoEncoder::publisher_publish(VideoEncoder *e, int segment_num, uint32_t
   edata.setFlags(flags);
   edata.setLen(dat.size());
   edat.setData(dat);
+  edat.setWidth(out_width);
+  edat.setHeight(out_height);
   if (flags & V4L2_BUF_FLAG_KEYFRAME) edat.setHeader(header);
 
   uint32_t bytes_size = capnp::computeSerializedSizeInWords(msg) * sizeof(capnp::word);

system/loggerd/encoder/encoder.h

@@ -22,10 +22,11 @@ public:
   virtual void encoder_open(const char* path) = 0;
   virtual void encoder_close() = 0;
 
-  static void publisher_publish(VideoEncoder *e, int segment_num, uint32_t idx, VisionIpcBufExtra &extra, unsigned int flags, kj::ArrayPtr<capnp::byte> header, kj::ArrayPtr<capnp::byte> dat);
+  void publisher_publish(VideoEncoder *e, int segment_num, uint32_t idx, VisionIpcBufExtra &extra, unsigned int flags, kj::ArrayPtr<capnp::byte> header, kj::ArrayPtr<capnp::byte> dat);
 
 protected:
   int in_width, in_height;
+  int out_width, out_height;
   const EncoderInfo encoder_info;
 
 private:

system/loggerd/encoder/ffmpeg_encoder.cc

@@ -29,16 +29,16 @@ FfmpegEncoder::FfmpegEncoder(const EncoderInfo &encoder_info, int in_width, int
   frame = av_frame_alloc();
   assert(frame);
   frame->format = AV_PIX_FMT_YUV420P;
-  frame->width = encoder_info.frame_width;
+  frame->width = out_width;
-  frame->height = encoder_info.frame_height;
+  frame->height = out_height;
-  frame->linesize[0] = encoder_info.frame_width;
+  frame->linesize[0] = out_width;
-  frame->linesize[1] = encoder_info.frame_width/2;
+  frame->linesize[1] = out_width/2;
-  frame->linesize[2] = encoder_info.frame_width/2;
+  frame->linesize[2] = out_width/2;
 
   convert_buf.resize(in_width * in_height * 3 / 2);
 
-  if (in_width != encoder_info.frame_width || in_height != encoder_info.frame_height) {
+  if (in_width != out_width || in_height != out_height) {
-    downscale_buf.resize(encoder_info.frame_width * encoder_info.frame_height * 3 / 2);
+    downscale_buf.resize(out_width * out_height * 3 / 2);
   }
 }
 

system/loggerd/encoder/v4l_encoder.cc

@@ -164,8 +164,8 @@ V4LEncoder::V4LEncoder(const EncoderInfo &encoder_info, int in_width, int in_hei
     .fmt = {
       .pix_mp = {
         // downscales are free with v4l
-        .width = (unsigned int)encoder_info.frame_width,
+        .width = (unsigned int)(out_width),
-        .height = (unsigned int)encoder_info.frame_height,
+        .height = (unsigned int)(out_height),
         .pixelformat = (encoder_info.encode_type == cereal::EncodeIndex::Type::FULL_H_E_V_C) ? V4L2_PIX_FMT_HEVC : V4L2_PIX_FMT_H264,
         .field = V4L2_FIELD_ANY,
         .colorspace = V4L2_COLORSPACE_DEFAULT,

system/loggerd/loggerd.cc

@@ -116,7 +116,7 @@ int handle_encoder_msg(LoggerdState *s, Message *msg, std::string &name, struct
           assert(encoder_info.filename != NULL);
           re.writer.reset(new VideoWriter(s->logger.segmentPath().c_str(),
             encoder_info.filename, idx.getType() != cereal::EncodeIndex::Type::FULL_H_E_V_C,
-            encoder_info.frame_width, encoder_info.frame_height, encoder_info.fps, idx.getType()));
+            edata.getWidth(), edata.getHeight(), encoder_info.fps, idx.getType()));
           // write the header
           auto header = edata.getHeader();
           re.writer->write((uint8_t *)header.begin(), header.size(), idx.getTimestampEof()/1000, true, false);

system/loggerd/loggerd.h

@@ -35,8 +35,8 @@ public:
   const char *publish_name;
   const char *filename = NULL;
   bool record = true;
-  int frame_width = 1928;
+  int frame_width = -1;
-  int frame_height = 1208;
+  int frame_height = -1;
   int fps = MAIN_FPS;
   int bitrate = MAIN_BITRATE;
   cereal::EncodeIndex::Type encode_type = Hardware::PC() ? cereal::EncodeIndex::Type::BIG_BOX_LOSSLESS

tools/sim/bridge/common.py

@@ -2,7 +2,7 @@ import signal
 import threading
 import functools
 
-from multiprocessing import Process, Queue
+from multiprocessing import Process, Queue, Value
 from abc import ABC, abstractmethod
 from typing import Optional
 
@@ -39,7 +39,7 @@ class SimulatorBridge(ABC):
     self._exit_event = threading.Event()
     self._threads = []
     self._keep_alive = True
-    self.started = False
+    self.started = Value('i', False)
     signal.signal(signal.SIGTERM, self._on_shutdown)
     self._exit = threading.Event()
     self.simulator_state = SimulatorState()
@@ -61,7 +61,7 @@ class SimulatorBridge(ABC):
       self.close()
 
   def close(self):
-    self.started = False
+    self.started.value = False
     self._exit_event.set()
 
     if self.world is not None:
@@ -181,6 +181,6 @@ Ignition: {self.simulator_state.ignition} Engaged: {self.simulator_state.is_enga
       if self.rk.frame % 25 == 0:
         self.print_status()
 
-      self.started = True
+      self.started.value = True
 
       self.rk.keep_time()

tools/sim/tests/test_metadrive_bridge.py

@@ -2,14 +2,13 @@
 import pytest
 import unittest
 
-from openpilot.tools.sim.run_bridge import parse_args
 from openpilot.tools.sim.bridge.metadrive.metadrive_bridge import MetaDriveBridge
 from openpilot.tools.sim.tests.test_sim_bridge import TestSimBridgeBase
 
 @pytest.mark.slow
 class TestMetaDriveBridge(TestSimBridgeBase):
   def create_bridge(self):
-    return MetaDriveBridge(parse_args([]))
+    return MetaDriveBridge(False, False)
 
 
 if __name__ == "__main__":

tools/sim/tests/test_sim_bridge.py

@@ -3,7 +3,7 @@ import subprocess
 import time
 import unittest
 
-from multiprocessing import Queue, Value
+from multiprocessing import Queue
 
 from cereal import messaging
 from openpilot.common.basedir import BASEDIR
@@ -27,7 +27,6 @@ class TestSimBridgeBase(unittest.TestCase):
     sm = messaging.SubMaster(['controlsState', 'onroadEvents', 'managerState'])
     q = Queue()
     bridge = self.create_bridge()
-    bridge.started = Value('b', False)
     p_bridge = bridge.run(q, retries=10)
     self.processes.append(p_bridge)
 
@@ -35,7 +34,7 @@ class TestSimBridgeBase(unittest.TestCase):
 
     # Wait for bridge to startup
     start_waiting = time.monotonic()
-    while not bridge.started and time.monotonic() < start_waiting + max_time_per_step:
+    while not bridge.started.value and time.monotonic() < start_waiting + max_time_per_step:
       time.sleep(0.1)
     self.assertEqual(p_bridge.exitcode, None, f"Bridge process should be running, but exited with code {p_bridge.exitcode}")