From 833316835aedaf259e6d6333a1be57cc31dce360 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Mon, 20 May 2024 11:48:51 -0500 Subject: [PATCH 01/26] [bot] Fingerprints: add missing FW versions from new users (#32486) Export fingerprints --- selfdrive/car/chrysler/fingerprints.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/selfdrive/car/chrysler/fingerprints.py b/selfdrive/car/chrysler/fingerprints.py index 95565ebf9d..a5b81650e3 100644 --- a/selfdrive/car/chrysler/fingerprints.py +++ b/selfdrive/car/chrysler/fingerprints.py @@ -68,6 +68,7 @@ FW_VERSIONS = { (Ecu.engine, 0x7e0, None): [ b'68267018AO ', b'68267020AJ ', + b'68303534AG ', b'68303534AJ ', b'68340762AD ', b'68340764AD ', @@ -106,6 +107,7 @@ FW_VERSIONS = { b'68405565AB', b'68405565AC', b'68444299AC', + b'68480707AC', b'68480708AC', b'68526663AB', ], @@ -147,6 +149,7 @@ FW_VERSIONS = { b'68443120AE ', b'68443123AC ', b'68443125AC ', + b'68496647AI ', b'68496647AJ ', b'68496650AH ', b'68496650AI ', From f3f22a56980d7a3fb2cd7c07fec8ac0e9e376a38 Mon Sep 17 00:00:00 2001 From: Hoang Bui <47828508+bongbui321@users.noreply.github.com> Date: Mon, 20 May 2024 17:41:31 -0400 Subject: [PATCH 02/26] CI/simulator: Fix metadrive test `pyopencl.CompilerWarning` (#32487) * add pytest.mark.filterwarnings * remove -W pyopencl.CompilerWarning * add comment --- .github/workflows/tools_tests.yaml | 2 +- tools/sim/tests/test_metadrive_bridge.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tools_tests.yaml b/.github/workflows/tools_tests.yaml index 4466260af1..c2e900fb12 100644 --- a/.github/workflows/tools_tests.yaml +++ b/.github/workflows/tools_tests.yaml @@ -59,7 +59,7 @@ jobs: ${{ env.RUN }} "export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && \ source selfdrive/test/setup_xvfb.sh && \ source selfdrive/test/setup_vsound.sh && \ - CI=1 pytest tools/sim/tests/test_metadrive_bridge.py -W ignore::pyopencl.CompilerWarning" + CI=1 pytest tools/sim/tests/test_metadrive_bridge.py" devcontainer: name: devcontainer diff --git a/tools/sim/tests/test_metadrive_bridge.py b/tools/sim/tests/test_metadrive_bridge.py index b53f4524a0..37bae63780 100755 --- a/tools/sim/tests/test_metadrive_bridge.py +++ b/tools/sim/tests/test_metadrive_bridge.py @@ -8,6 +8,7 @@ from openpilot.tools.sim.bridge.metadrive.metadrive_bridge import MetaDriveBridg from openpilot.tools.sim.tests.test_sim_bridge import TestSimBridgeBase @pytest.mark.slow +@pytest.mark.filterwarnings("ignore::pyopencl.CompilerWarning") # Unimportant warning of non-empty compile log class TestMetaDriveBridge(TestSimBridgeBase): def create_bridge(self): return MetaDriveBridge(False, False) From 3279dbeff7d4f696804c94de1573be2b5f3f2f9d Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Mon, 20 May 2024 14:59:43 -0700 Subject: [PATCH 03/26] athena: move to system/ (#32488) * athena: move to system/ * slash --- pyproject.toml | 2 +- release/files_common | 8 ++++---- selfdrive/manager/manager.py | 2 +- selfdrive/manager/process_config.py | 2 +- selfdrive/sentry.py | 2 +- {selfdrive => system}/athena/__init__.py | 0 {selfdrive => system}/athena/athenad.py | 0 {selfdrive => system}/athena/manage_athenad.py | 2 +- {selfdrive => system}/athena/registration.py | 0 {selfdrive => system}/athena/tests/__init__.py | 0 {selfdrive => system}/athena/tests/helpers.py | 0 {selfdrive => system}/athena/tests/test_athenad.py | 8 ++++---- .../athena/tests/test_athenad_ping.py | 4 ++-- .../athena/tests/test_registration.py | 12 ++++++------ 14 files changed, 21 insertions(+), 21 deletions(-) rename {selfdrive => system}/athena/__init__.py (100%) rename {selfdrive => system}/athena/athenad.py (100%) rename {selfdrive => system}/athena/manage_athenad.py (92%) rename {selfdrive => system}/athena/registration.py (100%) rename {selfdrive => system}/athena/tests/__init__.py (100%) rename {selfdrive => system}/athena/tests/helpers.py (100%) rename {selfdrive => system}/athena/tests/test_athenad.py (97%) rename {selfdrive => system}/athena/tests/test_athenad_ping.py (95%) rename {selfdrive => system}/athena/tests/test_registration.py (80%) diff --git a/pyproject.toml b/pyproject.toml index 544f9ee18c..2578d0a853 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,6 @@ markers = [ ] testpaths = [ "common", - "selfdrive/athena", "selfdrive/boardd", "selfdrive/car", "selfdrive/controls", @@ -31,6 +30,7 @@ testpaths = [ "selfdrive/test/longitudinal_maneuvers", "selfdrive/test/process_replay/test_fuzzy.py", "selfdrive/updated", + "system/athena", "system/camerad", "system/hardware/tici", "system/loggerd", diff --git a/release/files_common b/release/files_common index 34d8f00973..7dfccceb82 100644 --- a/release/files_common +++ b/release/files_common @@ -65,10 +65,10 @@ system/version.py selfdrive/SConscript -selfdrive/athena/__init__.py -selfdrive/athena/athenad.py -selfdrive/athena/manage_athenad.py -selfdrive/athena/registration.py +system/athena/__init__.py +system/athena/athenad.py +system/athena/manage_athenad.py +system/athena/registration.py selfdrive/boardd/.gitignore selfdrive/boardd/SConscript diff --git a/selfdrive/manager/manager.py b/selfdrive/manager/manager.py index 512320a8db..33c9ce9f5b 100755 --- a/selfdrive/manager/manager.py +++ b/selfdrive/manager/manager.py @@ -14,7 +14,7 @@ from openpilot.system.hardware import HARDWARE, PC from openpilot.selfdrive.manager.helpers import unblock_stdout, write_onroad_params, save_bootlog from openpilot.selfdrive.manager.process import ensure_running from openpilot.selfdrive.manager.process_config import managed_processes -from openpilot.selfdrive.athena.registration import register, UNREGISTERED_DONGLE_ID +from openpilot.system.athena.registration import register, UNREGISTERED_DONGLE_ID from openpilot.common.swaglog import cloudlog, add_file_handler from openpilot.system.version import get_build_metadata, terms_version, training_version diff --git a/selfdrive/manager/process_config.py b/selfdrive/manager/process_config.py index 4c34e7339e..5a501c2160 100644 --- a/selfdrive/manager/process_config.py +++ b/selfdrive/manager/process_config.py @@ -42,7 +42,7 @@ def only_offroad(started, params, CP: car.CarParams) -> bool: return not started procs = [ - DaemonProcess("manage_athenad", "selfdrive.athena.manage_athenad", "AthenadPid"), + DaemonProcess("manage_athenad", "system.athena.manage_athenad", "AthenadPid"), NativeProcess("camerad", "system/camerad", ["./camerad"], driverview), NativeProcess("logcatd", "system/logcatd", ["./logcatd"], only_onroad), diff --git a/selfdrive/sentry.py b/selfdrive/sentry.py index 204d9cab09..5f4772fa71 100644 --- a/selfdrive/sentry.py +++ b/selfdrive/sentry.py @@ -4,7 +4,7 @@ from enum import Enum from sentry_sdk.integrations.threading import ThreadingIntegration from openpilot.common.params import Params -from openpilot.selfdrive.athena.registration import is_registered_device +from openpilot.system.athena.registration import is_registered_device from openpilot.system.hardware import HARDWARE, PC from openpilot.common.swaglog import cloudlog from openpilot.system.version import get_build_metadata, get_version diff --git a/selfdrive/athena/__init__.py b/system/athena/__init__.py similarity index 100% rename from selfdrive/athena/__init__.py rename to system/athena/__init__.py diff --git a/selfdrive/athena/athenad.py b/system/athena/athenad.py similarity index 100% rename from selfdrive/athena/athenad.py rename to system/athena/athenad.py diff --git a/selfdrive/athena/manage_athenad.py b/system/athena/manage_athenad.py similarity index 92% rename from selfdrive/athena/manage_athenad.py rename to system/athena/manage_athenad.py index 2ec92cc3e0..853a7cd953 100755 --- a/selfdrive/athena/manage_athenad.py +++ b/system/athena/manage_athenad.py @@ -28,7 +28,7 @@ def main(): try: while 1: cloudlog.info("starting athena daemon") - proc = Process(name='athenad', target=launcher, args=('selfdrive.athena.athenad', 'athenad')) + proc = Process(name='athenad', target=launcher, args=('system.athena.athenad', 'athenad')) proc.start() proc.join() cloudlog.event("athenad exited", exitcode=proc.exitcode) diff --git a/selfdrive/athena/registration.py b/system/athena/registration.py similarity index 100% rename from selfdrive/athena/registration.py rename to system/athena/registration.py diff --git a/selfdrive/athena/tests/__init__.py b/system/athena/tests/__init__.py similarity index 100% rename from selfdrive/athena/tests/__init__.py rename to system/athena/tests/__init__.py diff --git a/selfdrive/athena/tests/helpers.py b/system/athena/tests/helpers.py similarity index 100% rename from selfdrive/athena/tests/helpers.py rename to system/athena/tests/helpers.py diff --git a/selfdrive/athena/tests/test_athenad.py b/system/athena/tests/test_athenad.py similarity index 97% rename from selfdrive/athena/tests/test_athenad.py rename to system/athena/tests/test_athenad.py index bdce3dccef..895cd8eb56 100755 --- a/selfdrive/athena/tests/test_athenad.py +++ b/system/athena/tests/test_athenad.py @@ -19,9 +19,9 @@ from cereal import messaging from openpilot.common.params import Params from openpilot.common.timeout import Timeout -from openpilot.selfdrive.athena import athenad -from openpilot.selfdrive.athena.athenad import MAX_RETRY_COUNT, dispatcher -from openpilot.selfdrive.athena.tests.helpers import HTTPRequestHandler, MockWebsocket, MockApi, EchoSocket +from openpilot.system.athena import athenad +from openpilot.system.athena.athenad import MAX_RETRY_COUNT, dispatcher +from openpilot.system.athena.tests.helpers import HTTPRequestHandler, MockWebsocket, MockApi, EchoSocket from openpilot.selfdrive.test.helpers import http_server_context from openpilot.system.hardware.hw import Paths @@ -50,7 +50,7 @@ def with_upload_handler(func): @pytest.fixture def mock_create_connection(mocker): - return mocker.patch('openpilot.selfdrive.athena.athenad.create_connection') + return mocker.patch('openpilot.system.athena.athenad.create_connection') @pytest.fixture def host(): diff --git a/selfdrive/athena/tests/test_athenad_ping.py b/system/athena/tests/test_athenad_ping.py similarity index 95% rename from selfdrive/athena/tests/test_athenad_ping.py rename to system/athena/tests/test_athenad_ping.py index 44fa0b8481..3152ca4271 100755 --- a/selfdrive/athena/tests/test_athenad_ping.py +++ b/system/athena/tests/test_athenad_ping.py @@ -7,7 +7,7 @@ from typing import cast from openpilot.common.params import Params from openpilot.common.timeout import Timeout -from openpilot.selfdrive.athena import athenad +from openpilot.system.athena import athenad from openpilot.selfdrive.manager.helpers import write_onroad_params from openpilot.system.hardware import TICI @@ -59,7 +59,7 @@ class TestAthenadPing: def assertTimeout(self, reconnect_time: float, subtests, mocker) -> None: self.athenad.start() - mock_create_connection = mocker.patch('openpilot.selfdrive.athena.athenad.create_connection', + mock_create_connection = mocker.patch('openpilot.system.athena.athenad.create_connection', new_callable=lambda: mocker.MagicMock(wraps=athenad.create_connection)) time.sleep(1) diff --git a/selfdrive/athena/tests/test_registration.py b/system/athena/tests/test_registration.py similarity index 80% rename from selfdrive/athena/tests/test_registration.py rename to system/athena/tests/test_registration.py index a808dd5668..85c065c1bf 100755 --- a/selfdrive/athena/tests/test_registration.py +++ b/system/athena/tests/test_registration.py @@ -4,8 +4,8 @@ from Crypto.PublicKey import RSA from pathlib import Path from openpilot.common.params import Params -from openpilot.selfdrive.athena.registration import register, UNREGISTERED_DONGLE_ID -from openpilot.selfdrive.athena.tests.helpers import MockResponse +from openpilot.system.athena.registration import register, UNREGISTERED_DONGLE_ID +from openpilot.system.athena.tests.helpers import MockResponse from openpilot.system.hardware.hw import Paths @@ -36,7 +36,7 @@ class TestRegistration: self.params.put("HardwareSerial", "serial") self._generate_keys() - m = mocker.patch("openpilot.selfdrive.athena.registration.api_get", autospec=True) + m = mocker.patch("openpilot.system.athena.registration.api_get", autospec=True) dongle = "DONGLE_ID_123" self.params.put("DongleId", dongle) assert register() == dongle @@ -44,7 +44,7 @@ class TestRegistration: def test_no_keys(self, mocker): # missing pubkey - m = mocker.patch("openpilot.selfdrive.athena.registration.api_get", autospec=True) + m = mocker.patch("openpilot.system.athena.registration.api_get", autospec=True) dongle = register() assert m.call_count == 0 assert dongle == UNREGISTERED_DONGLE_ID @@ -53,7 +53,7 @@ class TestRegistration: def test_missing_cache(self, mocker): # keys exist but no dongle id self._generate_keys() - m = mocker.patch("openpilot.selfdrive.athena.registration.api_get", autospec=True) + m = mocker.patch("openpilot.system.athena.registration.api_get", autospec=True) dongle = "DONGLE_ID_123" m.return_value = MockResponse(json.dumps({'dongle_id': dongle}), 200) assert register() == dongle @@ -67,7 +67,7 @@ class TestRegistration: def test_unregistered(self, mocker): # keys exist, but unregistered self._generate_keys() - m = mocker.patch("openpilot.selfdrive.athena.registration.api_get", autospec=True) + m = mocker.patch("openpilot.system.athena.registration.api_get", autospec=True) m.return_value = MockResponse(None, 402) dongle = register() assert m.call_count == 1 From 19c8e72c7494307c311fd2fc0cdb229e89a7ed0e Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Mon, 20 May 2024 15:50:54 -0700 Subject: [PATCH 04/26] Update BOUNTIES.md --- docs/BOUNTIES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/BOUNTIES.md b/docs/BOUNTIES.md index 7e7ee3b8d3..9d0718c572 100644 --- a/docs/BOUNTIES.md +++ b/docs/BOUNTIES.md @@ -9,6 +9,7 @@ Get paid to improve openpilot! * once you open a PR, the bounty is locked to you until you stop working on it * open a ticket at [comma.ai/support](https://comma.ai/support/shop-order) with links to your PRs to claim * get an extra 20% if you redeem your bounty in [comma shop](https://comma.ai/shop) credit (including refunds on previous orders) +* for bounties >$100, the first PR gets a lock, which times out after a week of no progress We put up each bounty with the intention that it'll get merged, but occasionally the right resolution is to close the bounty, which only becomes clear once some effort is put in. This is still valuable work, so we'll pay out $100 for getting any bounty closed with a good explanation. From 332542fad842a4d12ef289c89d5e8a8709c3c7a7 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Mon, 20 May 2024 16:01:17 -0700 Subject: [PATCH 05/26] cleanup old ignore paths --- .gitignore | 4 ---- .pre-commit-config.yaml | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 98fc70d029..693c532c54 100644 --- a/.gitignore +++ b/.gitignore @@ -55,12 +55,8 @@ selfdrive/modeld/_modeld selfdrive/modeld/_dmonitoringmodeld /src/ -one notebooks -xx -yy hyperthneed -panda_jungle provisioning .coverage* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 692c694560..c08ab1d5d1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,7 +47,7 @@ repos: args: - --local-partial-types - --explicit-package-bases - exclude: '^(third_party/)|(cereal/)|(opendbc/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)|(xx/)' + exclude: '^(third_party/)|(cereal/)|(opendbc/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)' - repo: local hooks: - id: cppcheck From af991b143ae4b89a15217cee77fcf8e93ec2508f Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Mon, 20 May 2024 16:46:34 -0700 Subject: [PATCH 06/26] fix pre-commit warning --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c08ab1d5d1..190425285d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -89,7 +89,7 @@ repos: entry: selfdrive/ui/tests/test_translations.py language: script pass_filenames: false - files: 'selfdrive/ui/translations/*' + files: '^selfdrive/ui/translations/' - repo: https://github.com/python-poetry/poetry rev: '1.8.0' hooks: From e0fa26b1a44bf54388392d54c8437459d870d78c Mon Sep 17 00:00:00 2001 From: macdoos <127897805+macdoos@users.noreply.github.com> Date: Tue, 21 May 2024 02:35:33 +0200 Subject: [PATCH 07/26] split out dev apt dependencies (#32476) * init * add more extra packages * update Dockerfile * cleanup * Update Dockerfile.openpilot_base * needed to build --------- Co-authored-by: Adeeb Shihadeh --- Dockerfile.openpilot_base | 12 ++++------ tools/install_ubuntu_dependencies.sh | 35 +++++++++++++++++++++------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base index a6c4c71790..0789a39c3e 100644 --- a/Dockerfile.openpilot_base +++ b/Dockerfile.openpilot_base @@ -13,14 +13,10 @@ ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 COPY tools/install_ubuntu_dependencies.sh /tmp/tools/ -RUN cd /tmp && \ - tools/install_ubuntu_dependencies.sh && \ - rm -rf /var/lib/apt/lists/* && \ - rm -rf /tmp/* && \ - # remove unused architectures from gcc for panda +RUN INSTALL_EXTRA_PACKAGES=no /tmp/tools/install_ubuntu_dependencies.sh && \ + rm -rf /var/lib/apt/lists/* /tmp/* && \ cd /usr/lib/gcc/arm-none-eabi/9.2.1 && \ - rm -rf arm/ && \ - rm -rf thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp + rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp # Add OpenCL RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -83,4 +79,4 @@ RUN cd /tmp && \ rm -rf /home/$USER/pyenv/versions/3.11.4/lib/python3.11/test USER root -RUN sudo git config --global --add safe.directory /tmp/openpilot \ No newline at end of file +RUN sudo git config --global --add safe.directory /tmp/openpilot diff --git a/tools/install_ubuntu_dependencies.sh b/tools/install_ubuntu_dependencies.sh index 72402b3bf7..7abfbf51c8 100755 --- a/tools/install_ubuntu_dependencies.sh +++ b/tools/install_ubuntu_dependencies.sh @@ -12,21 +12,17 @@ if [[ ! $(id -u) -eq 0 ]]; then SUDO="sudo" fi -# Install packages present in all supported versions of Ubuntu +# Install common packages function install_ubuntu_common_requirements() { $SUDO apt-get update $SUDO apt-get install -y --no-install-recommends \ autoconf \ build-essential \ ca-certificates \ - casync \ clang \ - cmake \ - make \ cppcheck \ libtool \ gcc-arm-none-eabi \ - bzip2 \ liblzma-dev \ libarchive-dev \ libbz2-dev \ @@ -62,9 +58,7 @@ function install_ubuntu_common_requirements() { opencl-headers \ ocl-icd-libopencl1 \ ocl-icd-opencl-dev \ - clinfo \ portaudio19-dev \ - qml-module-qtquick2 \ qtmultimedia5-dev \ qtlocation5-dev \ qtpositioning5-dev \ @@ -75,7 +69,18 @@ function install_ubuntu_common_requirements() { libqt5serialbus5-dev \ libqt5x11extras5-dev \ libqt5opengl5-dev \ - libreadline-dev \ + libreadline-dev +} + +# Install extra packages +function install_extra_packages() { + echo "Installing extra packages..." + $SUDO apt-get install -y --no-install-recommends \ + bzip2 \ + clinfo \ + casync \ + cmake \ + make \ libdw1 } @@ -125,7 +130,19 @@ if [ -f "/etc/os-release" ]; then install_ubuntu_lts_latest_requirements fi esac + + # Install extra packages + if [[ -z "$INSTALL_EXTRA_PACKAGES" ]]; then + read -p "Base setup done. Do you want to install extra development packages? [Y/n]: " -n 1 -r + echo "" + if [[ $REPLY =~ ^[Yy]$ ]]; then + INSTALL_EXTRA_PACKAGES="yes" + fi + fi + if [[ "$INSTALL_EXTRA_PACKAGES" == "yes" ]]; then + install_extra_packages + fi else - echo "No /etc/os-release in the system" + echo "No /etc/os-release in the system. Make sure you're running on Ubuntu, or similar." exit 1 fi From 1ea2411575a04176637856af5ce54aae01b1cd8e Mon Sep 17 00:00:00 2001 From: LostEon <86137295+LostEon@users.noreply.github.com> Date: Mon, 20 May 2024 18:38:47 -0600 Subject: [PATCH 08/26] Fingerprint 2016 Jeep Grand Cherokee (#32491) * Fingerprint 2016 Grand Cherokee * sort --------- Co-authored-by: Shane Smiskol --- selfdrive/car/chrysler/fingerprints.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/selfdrive/car/chrysler/fingerprints.py b/selfdrive/car/chrysler/fingerprints.py index a5b81650e3..b974d3dd4d 100644 --- a/selfdrive/car/chrysler/fingerprints.py +++ b/selfdrive/car/chrysler/fingerprints.py @@ -267,6 +267,7 @@ FW_VERSIONS = { }, CAR.JEEP_GRAND_CHEROKEE: { (Ecu.combinationMeter, 0x742, None): [ + b'68243549AG', b'68302211AC', b'68302212AD', b'68302223AC', @@ -278,18 +279,22 @@ FW_VERSIONS = { b'68340272AD', ], (Ecu.srs, 0x744, None): [ + b'68309533AA', b'68316742AB', b'68355363AB', ], (Ecu.abs, 0x747, None): [ + b'68252642AG', b'68306178AD', b'68336276AB', ], (Ecu.fwdRadar, 0x753, None): [ b'04672627AB', + b'68251506AF', b'68332015AB', ], (Ecu.eps, 0x75a, None): [ + b'68276201AG', b'68321644AB', b'68321644AC', b'68321646AC', @@ -297,6 +302,7 @@ FW_VERSIONS = { ], (Ecu.engine, 0x7e0, None): [ b'05035920AE ', + b'68252272AG ', b'68284455AI ', b'68284456AI ', b'68284477AF ', @@ -307,6 +313,7 @@ FW_VERSIONS = { ], (Ecu.transmission, 0x7e1, None): [ b'05035517AH', + b'68253222AF', b'68311218AC', b'68311223AF', b'68311223AG', From 788c4edeecc0f35d8c74f486f664c1835098cdf4 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Mon, 20 May 2024 17:40:46 -0700 Subject: [PATCH 09/26] ruff: enable NPY --- pyproject.toml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2578d0a853..e445df0479 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -184,8 +184,16 @@ build-backend = "poetry.core.masonry.api" # https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml [tool.ruff] indent-width = 2 -lint.select = ["E", "F", "W", "PIE", "C4", "ISC", "RUF008", "RUF100", "A", "B", "TID251"] -lint.ignore = ["E741", "E402", "C408", "ISC003", "B027", "B024"] +lint.select = ["E", "F", "W", "PIE", "C4", "ISC", "NPY", "RUF008", "RUF100", "A", "B", "TID251"] +lint.ignore = [ + "E741", + "E402", + "C408", + "ISC003", + "B027", + "B024", + "NPY002", # new numpy random syntax is worse +] line-length = 160 target-version="py311" exclude = [ From b2e4c64cf8fe3256d1e411e8715c29e2eb6f4f00 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Mon, 20 May 2024 17:43:54 -0700 Subject: [PATCH 10/26] ruff: enable UP --- common/api/__init__.py | 2 +- common/gpio.py | 4 ++-- common/spinner.py | 2 +- common/stat_live.py | 4 ++-- pyproject.toml | 3 ++- selfdrive/controls/lib/lateral_mpc_lib/lat_mpc.py | 2 +- selfdrive/controls/lib/pid.py | 2 +- selfdrive/controls/radard.py | 4 ++-- selfdrive/controls/tests/test_following_distance.py | 2 +- selfdrive/debug/vw_mqb_config.py | 2 +- selfdrive/locationd/models/car_kf.py | 2 +- selfdrive/locationd/models/live_kf.py | 4 ++-- selfdrive/modeld/modeld.py | 5 ++--- selfdrive/monitoring/driver_monitor.py | 8 ++++---- selfdrive/statsd.py | 4 ++-- selfdrive/test/longitudinal_maneuvers/plant.py | 2 +- selfdrive/test/process_replay/test_imgproc.py | 4 ++-- selfdrive/test/profiling/lib.py | 4 ++-- system/athena/tests/helpers.py | 6 +++--- system/loggerd/tests/loggerd_tests_common.py | 6 +++--- system/ubloxd/pigeond.py | 2 +- system/updated/casync/tar.py | 3 ++- system/version.py | 2 +- system/webrtc/tests/test_webrtcd.py | 2 +- tools/lib/api.py | 2 +- 25 files changed, 42 insertions(+), 41 deletions(-) diff --git a/common/api/__init__.py b/common/api/__init__.py index 79875023a2..f3a7d83842 100644 --- a/common/api/__init__.py +++ b/common/api/__init__.py @@ -7,7 +7,7 @@ from openpilot.system.version import get_version API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com') -class Api(): +class Api: def __init__(self, dongle_id): self.dongle_id = dongle_id with open(Paths.persist_root()+'/comma/id_rsa') as f: diff --git a/common/gpio.py b/common/gpio.py index 68932cb87a..66b2adf438 100644 --- a/common/gpio.py +++ b/common/gpio.py @@ -1,5 +1,5 @@ import os -from functools import lru_cache +from functools import cache def gpio_init(pin: int, output: bool) -> None: try: @@ -35,7 +35,7 @@ def gpio_export(pin: int) -> None: except Exception: print(f"Failed to export gpio {pin}") -@lru_cache(maxsize=None) +@cache def get_irq_action(irq: int) -> list[str]: try: with open(f"/sys/kernel/irq/{irq}/actions") as f: diff --git a/common/spinner.py b/common/spinner.py index 43d4bb2cc2..dcf22641c4 100644 --- a/common/spinner.py +++ b/common/spinner.py @@ -3,7 +3,7 @@ import subprocess from openpilot.common.basedir import BASEDIR -class Spinner(): +class Spinner: def __init__(self): try: self.spinner_proc = subprocess.Popen(["./spinner"], diff --git a/common/stat_live.py b/common/stat_live.py index a91c1819bb..3901c448d8 100644 --- a/common/stat_live.py +++ b/common/stat_live.py @@ -1,6 +1,6 @@ import numpy as np -class RunningStat(): +class RunningStat: # tracks realtime mean and standard deviation without storing any data def __init__(self, priors=None, max_trackable=-1): self.max_trackable = max_trackable @@ -51,7 +51,7 @@ class RunningStat(): def params_to_save(self): return [self.M, self.S, self.n] -class RunningStatFilter(): +class RunningStatFilter: def __init__(self, raw_priors=None, filtered_priors=None, max_trackable=-1): self.raw_stat = RunningStat(raw_priors, -1) self.filtered_stat = RunningStat(filtered_priors, max_trackable) diff --git a/pyproject.toml b/pyproject.toml index e445df0479..ce3cf4b6c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -184,7 +184,7 @@ build-backend = "poetry.core.masonry.api" # https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml [tool.ruff] indent-width = 2 -lint.select = ["E", "F", "W", "PIE", "C4", "ISC", "NPY", "RUF008", "RUF100", "A", "B", "TID251"] +lint.select = ["E", "F", "W", "PIE", "C4", "ISC", "NPY", "UP", "RUF008", "RUF100", "A", "B", "TID251"] lint.ignore = [ "E741", "E402", @@ -193,6 +193,7 @@ lint.ignore = [ "B027", "B024", "NPY002", # new numpy random syntax is worse + "UP038", # (x, y) -> x|y for isinstance ] line-length = 160 target-version="py311" diff --git a/selfdrive/controls/lib/lateral_mpc_lib/lat_mpc.py b/selfdrive/controls/lib/lateral_mpc_lib/lat_mpc.py index 83ec3b3a13..ad60861088 100755 --- a/selfdrive/controls/lib/lateral_mpc_lib/lat_mpc.py +++ b/selfdrive/controls/lib/lateral_mpc_lib/lat_mpc.py @@ -128,7 +128,7 @@ def gen_lat_ocp(): return ocp -class LateralMpc(): +class LateralMpc: def __init__(self, x0=None): if x0 is None: x0 = np.zeros(X_DIM) diff --git a/selfdrive/controls/lib/pid.py b/selfdrive/controls/lib/pid.py index f4ec7e5996..29c4d8bd46 100644 --- a/selfdrive/controls/lib/pid.py +++ b/selfdrive/controls/lib/pid.py @@ -4,7 +4,7 @@ from numbers import Number from openpilot.common.numpy_fast import clip, interp -class PIDController(): +class PIDController: def __init__(self, k_p, k_i, k_f=0., k_d=0., pos_limit=1e308, neg_limit=-1e308, rate=100): self._k_p = k_p self._k_i = k_i diff --git a/selfdrive/controls/radard.py b/selfdrive/controls/radard.py index 16c9e0635c..7420f666f7 100755 --- a/selfdrive/controls/radard.py +++ b/selfdrive/controls/radard.py @@ -2,7 +2,7 @@ import importlib import math from collections import deque -from typing import Any, Optional +from typing import Any import capnp from cereal import messaging, log, car @@ -208,7 +208,7 @@ class RadarD: self.ready = False - def update(self, sm: messaging.SubMaster, rr: Optional[car.RadarData]): + def update(self, sm: messaging.SubMaster, rr): self.ready = sm.seen['modelV2'] self.current_time = 1e-9*max(sm.logMonoTime.values()) diff --git a/selfdrive/controls/tests/test_following_distance.py b/selfdrive/controls/tests/test_following_distance.py index 5d60911805..89a575a9ad 100755 --- a/selfdrive/controls/tests/test_following_distance.py +++ b/selfdrive/controls/tests/test_following_distance.py @@ -32,7 +32,7 @@ def run_following_distance_simulation(v_lead, t_end=100.0, e2e=False, personalit log.LongitudinalPersonality.standard, log.LongitudinalPersonality.aggressive], [0,10,35])) # speed -class TestFollowingDistance(): +class TestFollowingDistance: def test_following_distance(self): v_lead = float(self.speed) simulation_steady_state = run_following_distance_simulation(v_lead, e2e=self.e2e, personality=self.personality) diff --git a/selfdrive/debug/vw_mqb_config.py b/selfdrive/debug/vw_mqb_config.py index 64bc2bc638..6df1e57ce3 100755 --- a/selfdrive/debug/vw_mqb_config.py +++ b/selfdrive/debug/vw_mqb_config.py @@ -139,7 +139,7 @@ if __name__ == "__main__": # so fib a little and say that same tester did the programming current_date = date.today() formatted_date = current_date.strftime('%y-%m-%d') - year, month, day = [int(part) for part in formatted_date.split('-')] + year, month, day = (int(part) for part in formatted_date.split('-')) prog_date = bytes([year, month, day]) uds_client.write_data_by_identifier(DATA_IDENTIFIER_TYPE.PROGRAMMING_DATE, prog_date) tester_num = uds_client.read_data_by_identifier(DATA_IDENTIFIER_TYPE.CALIBRATION_REPAIR_SHOP_CODE_OR_CALIBRATION_EQUIPMENT_SERIAL_NUMBER) diff --git a/selfdrive/locationd/models/car_kf.py b/selfdrive/locationd/models/car_kf.py index 1f3e447a19..87a93cdd0c 100755 --- a/selfdrive/locationd/models/car_kf.py +++ b/selfdrive/locationd/models/car_kf.py @@ -28,7 +28,7 @@ def _slice(n): return s -class States(): +class States: # Vehicle model params STIFFNESS = _slice(1) # [-] STEER_RATIO = _slice(1) # [-] diff --git a/selfdrive/locationd/models/live_kf.py b/selfdrive/locationd/models/live_kf.py index c4933a6129..0cc3eca61d 100755 --- a/selfdrive/locationd/models/live_kf.py +++ b/selfdrive/locationd/models/live_kf.py @@ -20,7 +20,7 @@ def numpy2eigenstring(arr): return f"(Eigen::VectorXd({len(arr)}) << {arr_str}).finished()" -class States(): +class States: ECEF_POS = slice(0, 3) # x, y and z in ECEF in meters ECEF_ORIENTATION = slice(3, 7) # quat for pose of phone in ecef ECEF_VELOCITY = slice(7, 10) # ecef velocity in m/s @@ -39,7 +39,7 @@ class States(): ACC_BIAS_ERR = slice(18, 21) -class LiveKalman(): +class LiveKalman: name = 'live' initial_x = np.array([3.88e6, -3.37e6, 3.76e6, diff --git a/selfdrive/modeld/modeld.py b/selfdrive/modeld/modeld.py index 35e2a29aa1..66563a3c1c 100755 --- a/selfdrive/modeld/modeld.py +++ b/selfdrive/modeld/modeld.py @@ -206,9 +206,8 @@ def main(demo=False): continue if abs(meta_main.timestamp_sof - meta_extra.timestamp_sof) > 10000000: - cloudlog.error("frames out of sync! main: {} ({:.5f}), extra: {} ({:.5f})".format( - meta_main.frame_id, meta_main.timestamp_sof / 1e9, - meta_extra.frame_id, meta_extra.timestamp_sof / 1e9)) + cloudlog.error(f"frames out of sync! main: {meta_main.frame_id} ({meta_main.timestamp_sof / 1e9:.5f}),\ + extra: {meta_extra.frame_id} ({meta_extra.timestamp_sof / 1e9:.5f})") else: # Use single camera diff --git a/selfdrive/monitoring/driver_monitor.py b/selfdrive/monitoring/driver_monitor.py index 9faa5f4d05..85cc19eb8b 100644 --- a/selfdrive/monitoring/driver_monitor.py +++ b/selfdrive/monitoring/driver_monitor.py @@ -15,7 +15,7 @@ EventName = car.CarEvent.EventName # We recommend that you do not change these numbers from the defaults. # ****************************************************************************************** -class DRIVER_MONITOR_SETTINGS(): +class DRIVER_MONITOR_SETTINGS: def __init__(self): self._DT_DMON = DT_DMON # ref (page15-16): https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:42018X1947&rid=2 @@ -102,7 +102,7 @@ def face_orientation_from_net(angles_desc, pos_desc, rpy_calib): yaw -= rpy_calib[2] return roll_net, pitch, yaw -class DriverPose(): +class DriverPose: def __init__(self, max_trackable): self.yaw = 0. self.pitch = 0. @@ -116,12 +116,12 @@ class DriverPose(): self.cfactor_pitch = 1. self.cfactor_yaw = 1. -class DriverBlink(): +class DriverBlink: def __init__(self): self.left_blink = 0. self.right_blink = 0. -class DriverStatus(): +class DriverStatus: def __init__(self, rhd_saved=False, settings=None, always_on=False): if settings is None: settings = DRIVER_MONITOR_SETTINGS() diff --git a/selfdrive/statsd.py b/selfdrive/statsd.py index 2f9149b096..2b5a7bb3a7 100755 --- a/selfdrive/statsd.py +++ b/selfdrive/statsd.py @@ -4,7 +4,7 @@ import zmq import time from pathlib import Path from collections import defaultdict -from datetime import datetime, timezone +from datetime import datetime, UTC from typing import NoReturn from openpilot.common.params import Params @@ -133,7 +133,7 @@ def main() -> NoReturn: # flush when started state changes or after FLUSH_TIME_S if (time.monotonic() > last_flush_time + STATS_FLUSH_TIME_S) or (sm['deviceState'].started != started_prev): result = "" - current_time = datetime.utcnow().replace(tzinfo=timezone.utc) + current_time = datetime.utcnow().replace(tzinfo=UTC) tags['started'] = sm['deviceState'].started for key, value in gauges.items(): diff --git a/selfdrive/test/longitudinal_maneuvers/plant.py b/selfdrive/test/longitudinal_maneuvers/plant.py index ac54967f88..daf7cec32b 100755 --- a/selfdrive/test/longitudinal_maneuvers/plant.py +++ b/selfdrive/test/longitudinal_maneuvers/plant.py @@ -83,7 +83,7 @@ class Plant: lead = log.RadarState.LeadData.new_message() lead.dRel = float(d_rel) - lead.yRel = float(0.0) + lead.yRel = 0.0 lead.vRel = float(v_rel) lead.aRel = float(a_lead - self.acceleration) lead.vLead = float(v_lead) diff --git a/selfdrive/test/process_replay/test_imgproc.py b/selfdrive/test/process_replay/test_imgproc.py index a980548baa..036f21b9e5 100755 --- a/selfdrive/test/process_replay/test_imgproc.py +++ b/selfdrive/test/process_replay/test_imgproc.py @@ -93,7 +93,7 @@ if __name__ == "__main__": if pix_hash != ref_hash: print("result changed! please check kernel") - print("ref: %s" % ref_hash) - print("new: %s" % pix_hash) + print(f"ref: {ref_hash}") + print(f"new: {pix_hash}") else: print("test passed") diff --git a/selfdrive/test/profiling/lib.py b/selfdrive/test/profiling/lib.py index 7f3b0126ff..93ba215386 100644 --- a/selfdrive/test/profiling/lib.py +++ b/selfdrive/test/profiling/lib.py @@ -8,7 +8,7 @@ class ReplayDone(Exception): pass -class SubSocket(): +class SubSocket: def __init__(self, msgs, trigger): self.i = 0 self.trigger = trigger @@ -28,7 +28,7 @@ class SubSocket(): return msg -class PubSocket(): +class PubSocket: def send(self, data): pass diff --git a/system/athena/tests/helpers.py b/system/athena/tests/helpers.py index 322e9d81dd..a0a9cccdc1 100644 --- a/system/athena/tests/helpers.py +++ b/system/athena/tests/helpers.py @@ -9,7 +9,7 @@ class MockResponse: self.status_code = status_code -class EchoSocket(): +class EchoSocket: def __init__(self, port): self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.bind(('127.0.0.1', port)) @@ -34,7 +34,7 @@ class EchoSocket(): self.socket.close() -class MockApi(): +class MockApi: def __init__(self, dongle_id): pass @@ -42,7 +42,7 @@ class MockApi(): return "fake-token" -class MockWebsocket(): +class MockWebsocket: sock = socket.socket() def __init__(self, recv_queue, send_queue): diff --git a/system/loggerd/tests/loggerd_tests_common.py b/system/loggerd/tests/loggerd_tests_common.py index e8a6d031c4..15fd997eb8 100644 --- a/system/loggerd/tests/loggerd_tests_common.py +++ b/system/loggerd/tests/loggerd_tests_common.py @@ -28,12 +28,12 @@ def create_random_file(file_path: Path, size_mb: float, lock: bool = False, uplo if upload_xattr is not None: setxattr(str(file_path), uploader.UPLOAD_ATTR_NAME, upload_xattr) -class MockResponse(): +class MockResponse: def __init__(self, text, status_code): self.text = text self.status_code = status_code -class MockApi(): +class MockApi: def __init__(self, dongle_id): pass @@ -43,7 +43,7 @@ class MockApi(): def get_token(self): return "fake-token" -class MockApiIgnore(): +class MockApiIgnore: def __init__(self, dongle_id): pass diff --git a/system/ubloxd/pigeond.py b/system/ubloxd/pigeond.py index 21b3a86f97..5711992cfb 100755 --- a/system/ubloxd/pigeond.py +++ b/system/ubloxd/pigeond.py @@ -61,7 +61,7 @@ def get_assistnow_messages(token: bytes) -> list[bytes]: return msgs -class TTYPigeon(): +class TTYPigeon: def __init__(self): self.tty = serial.VTIMESerial(UBLOX_TTY, baudrate=9600, timeout=0) diff --git a/system/updated/casync/tar.py b/system/updated/casync/tar.py index 5c547e73a2..a5a8238bba 100644 --- a/system/updated/casync/tar.py +++ b/system/updated/casync/tar.py @@ -1,6 +1,7 @@ import pathlib import tarfile -from typing import IO, Callable +from typing import IO +from collections.abc import Callable def include_default(_) -> bool: diff --git a/system/version.py b/system/version.py index f5b8cd49e8..d258fe0492 100755 --- a/system/version.py +++ b/system/version.py @@ -27,7 +27,7 @@ def get_version(path: str = BASEDIR) -> str: def get_release_notes(path: str = BASEDIR) -> str: - with open(os.path.join(path, "RELEASES.md"), "r") as f: + with open(os.path.join(path, "RELEASES.md")) as f: return f.read().split('\n\n', 1)[0] diff --git a/system/webrtc/tests/test_webrtcd.py b/system/webrtc/tests/test_webrtcd.py index 684c7cf359..b0bc7b1966 100755 --- a/system/webrtc/tests/test_webrtcd.py +++ b/system/webrtc/tests/test_webrtcd.py @@ -20,7 +20,7 @@ from parameterized import parameterized_class ([], []), ]) @pytest.mark.asyncio -class TestWebrtcdProc(): +class TestWebrtcdProc: async def assertCompletesWithTimeout(self, awaitable, timeout=1): try: async with asyncio.timeout(timeout): diff --git a/tools/lib/api.py b/tools/lib/api.py index 6ff9242f29..92a75d2a3b 100644 --- a/tools/lib/api.py +++ b/tools/lib/api.py @@ -2,7 +2,7 @@ import os import requests API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com') -class CommaApi(): +class CommaApi: def __init__(self, token=None): self.session = requests.Session() self.session.headers['User-agent'] = 'OpenpilotTools' From 0e3df5ae4d673c3644f04861a73e3aa599dd03ae Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Mon, 20 May 2024 17:47:48 -0700 Subject: [PATCH 11/26] ruff: enable TRY --- pyproject.toml | 9 ++++++++- system/webrtc/webrtcd.py | 12 ++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ce3cf4b6c3..2b183c983e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -184,7 +184,14 @@ build-backend = "poetry.core.masonry.api" # https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml [tool.ruff] indent-width = 2 -lint.select = ["E", "F", "W", "PIE", "C4", "ISC", "NPY", "UP", "RUF008", "RUF100", "A", "B", "TID251"] +lint.select = [ + "E", "F", "W", "PIE", "C4", "ISC", "A", "B", + "NPY", # numpy + "UP", # pyupgrade + "TRY302", "TRY400", "TRY401", # try/excepts + "RUF008", "RUF100", + "TID251" +] lint.ignore = [ "E741", "E402", diff --git a/system/webrtc/webrtcd.py b/system/webrtc/webrtcd.py index 51c86aacc6..76c3cd8470 100755 --- a/system/webrtc/webrtcd.py +++ b/system/webrtc/webrtcd.py @@ -97,8 +97,8 @@ class CerealProxyRunner: except InvalidStateError: self.logger.warning("Cereal outgoing proxy invalid state (connection closed)") break - except Exception as ex: - self.logger.error("Cereal outgoing proxy failure: %s", ex) + except Exception: + self.logger.exception("Cereal outgoing proxy failure") await asyncio.sleep(0.01) @@ -175,8 +175,8 @@ class StreamSession: assert self.incoming_bridge is not None try: self.incoming_bridge.send(message) - except Exception as ex: - self.logger.error("Cereal incoming proxy failure: %s", ex) + except Exception: + self.logger.exception("Cereal incoming proxy failure") async def run(self): try: @@ -200,8 +200,8 @@ class StreamSession: await self.post_run_cleanup() self.logger.info("Stream session (%s) ended", self.identifier) - except Exception as ex: - self.logger.error("Stream session failure: %s", ex) + except Exception: + self.logger.exception("Stream session failure") async def post_run_cleanup(self): await self.stream.stop() From 0678644a8f1c5292d7904ecde6e55aeb90550b52 Mon Sep 17 00:00:00 2001 From: Dean Lee Date: Tue, 21 May 2024 10:38:24 +0800 Subject: [PATCH 12/26] ui: improve `update_line_data()` (#32354) improve update_line_data --- selfdrive/ui/ui.cc | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/selfdrive/ui/ui.cc b/selfdrive/ui/ui.cc index e5309c9690..c4ddd17e25 100644 --- a/selfdrive/ui/ui.cc +++ b/selfdrive/ui/ui.cc @@ -57,26 +57,23 @@ void update_leads(UIState *s, const cereal::RadarState::Reader &radar_state, con void update_line_data(const UIState *s, const cereal::XYZTData::Reader &line, float y_off, float z_off, QPolygonF *pvd, int max_idx, bool allow_invert=true) { const auto line_x = line.getX(), line_y = line.getY(), line_z = line.getZ(); - QPolygonF left_points, right_points; - left_points.reserve(max_idx + 1); - right_points.reserve(max_idx + 1); - + QPointF left, right; + pvd->clear(); for (int i = 0; i <= max_idx; i++) { // highly negative x positions are drawn above the frame and cause flickering, clip to zy plane of camera if (line_x[i] < 0) continue; - QPointF left, right; + bool l = calib_frame_to_full_frame(s, line_x[i], line_y[i] - y_off, line_z[i] + z_off, &left); bool r = calib_frame_to_full_frame(s, line_x[i], line_y[i] + y_off, line_z[i] + z_off, &right); if (l && r) { // For wider lines the drawn polygon will "invert" when going over a hill and cause artifacts - if (!allow_invert && left_points.size() && left.y() > left_points.back().y()) { + if (!allow_invert && pvd->size() && left.y() > pvd->back().y()) { continue; } - left_points.push_back(left); - right_points.push_front(right); + pvd->push_back(left); + pvd->push_front(right); } } - *pvd = left_points + right_points; } void update_model(UIState *s, From 1203f5eeb363f4ccdf149d6fa1c32380c0c90027 Mon Sep 17 00:00:00 2001 From: Mauricio Alvarez Leon <65101411+BBBmau@users.noreply.github.com> Date: Mon, 20 May 2024 20:47:53 -0700 Subject: [PATCH 13/26] minimize ubuntu deps. needed to run build openpilot (#32492) * minimize common deps list * fix * fix endline error * add portaudio3 * add build-essential * upload old loggerd deps * libqt5charts5-dev * libncurses5-dev * libbz2-dev * libsqlite3-dev --- tools/install_ubuntu_dependencies.sh | 33 ++++++++++++++-------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/tools/install_ubuntu_dependencies.sh b/tools/install_ubuntu_dependencies.sh index 7abfbf51c8..0e3d453ce4 100755 --- a/tools/install_ubuntu_dependencies.sh +++ b/tools/install_ubuntu_dependencies.sh @@ -16,16 +16,12 @@ fi function install_ubuntu_common_requirements() { $SUDO apt-get update $SUDO apt-get install -y --no-install-recommends \ - autoconf \ - build-essential \ ca-certificates \ clang \ cppcheck \ - libtool \ + build-essential \ gcc-arm-none-eabi \ liblzma-dev \ - libarchive-dev \ - libbz2-dev \ capnproto \ libcapnp-dev \ curl \ @@ -38,21 +34,21 @@ function install_ubuntu_common_requirements() { libavdevice-dev \ libavutil-dev \ libavfilter-dev \ + libbz2-dev \ libeigen3-dev \ libffi-dev \ libglew-dev \ libgles2-mesa-dev \ libglfw3-dev \ libglib2.0-0 \ - libncurses5-dev \ - libncursesw5-dev \ libomp-dev \ libpng16-16 \ - libportaudio2 \ + libqt5charts5-dev \ + libncurses5-dev \ libssl-dev \ - libsqlite3-dev \ libusb-1.0-0-dev \ libzmq3-dev \ + libsqlite3-dev \ libsystemd-dev \ locales \ opencl-headers \ @@ -63,25 +59,30 @@ function install_ubuntu_common_requirements() { qtlocation5-dev \ qtpositioning5-dev \ qttools5-dev-tools \ - libqt5sql5-sqlite \ libqt5svg5-dev \ - libqt5charts5-dev \ libqt5serialbus5-dev \ libqt5x11extras5-dev \ - libqt5opengl5-dev \ - libreadline-dev + libqt5opengl5-dev } # Install extra packages function install_extra_packages() { echo "Installing extra packages..." $SUDO apt-get install -y --no-install-recommends \ - bzip2 \ - clinfo \ casync \ cmake \ make \ - libdw1 + clinfo \ + libqt5sql5-sqlite \ + libreadline-dev \ + libdw1 \ + autoconf \ + libtool \ + bzip2 \ + libarchive-dev \ + libncursesw5-dev \ + libportaudio2 \ + locales } # Install Ubuntu 24.04 LTS packages From da6fd751257bcd3aa087ac4114e9f1529f7d3d12 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Mon, 20 May 2024 22:39:25 -0700 Subject: [PATCH 14/26] move sentry/stats to system/ (#32490) * move sentry/stats to system/ * fix --- release/files_common | 7 ++++--- selfdrive/manager/manager.py | 2 +- selfdrive/manager/process.py | 2 +- selfdrive/manager/process_config.py | 4 ++-- selfdrive/modeld/modeld.py | 2 +- selfdrive/test/test_onroad.py | 4 ++-- selfdrive/thermald/power_monitoring.py | 2 +- selfdrive/thermald/thermald.py | 2 +- {selfdrive => system}/sentry.py | 0 {selfdrive => system}/statsd.py | 0 {selfdrive => system}/tombstoned.py | 2 +- 11 files changed, 14 insertions(+), 13 deletions(-) rename {selfdrive => system}/sentry.py (100%) rename {selfdrive => system}/statsd.py (100%) rename {selfdrive => system}/tombstoned.py (99%) diff --git a/release/files_common b/release/files_common index 7dfccceb82..19a93bc7da 100644 --- a/release/files_common +++ b/release/files_common @@ -53,9 +53,6 @@ tools/replay/*.cc tools/replay/*.h selfdrive/__init__.py -selfdrive/sentry.py -selfdrive/tombstoned.py -selfdrive/statsd.py selfdrive/updated/** @@ -162,6 +159,10 @@ selfdrive/controls/lib/longitudinal_mpc_lib/* system/__init__.py system/*.py +system/sentry.py +system/tombstoned.py +system/statsd.py + system/hardware/__init__.py system/hardware/base.h system/hardware/base.py diff --git a/selfdrive/manager/manager.py b/selfdrive/manager/manager.py index 33c9ce9f5b..408314d62f 100755 --- a/selfdrive/manager/manager.py +++ b/selfdrive/manager/manager.py @@ -7,7 +7,7 @@ import traceback from cereal import log import cereal.messaging as messaging -import openpilot.selfdrive.sentry as sentry +import openpilot.system.sentry as sentry from openpilot.common.params import Params, ParamKeyType from openpilot.common.text_window import TextWindow from openpilot.system.hardware import HARDWARE, PC diff --git a/selfdrive/manager/process.py b/selfdrive/manager/process.py index 7964f5229d..36f299ae62 100644 --- a/selfdrive/manager/process.py +++ b/selfdrive/manager/process.py @@ -12,7 +12,7 @@ from setproctitle import setproctitle from cereal import car, log import cereal.messaging as messaging -import openpilot.selfdrive.sentry as sentry +import openpilot.system.sentry as sentry from openpilot.common.basedir import BASEDIR from openpilot.common.params import Params from openpilot.common.swaglog import cloudlog diff --git a/selfdrive/manager/process_config.py b/selfdrive/manager/process_config.py index 5a501c2160..5ea4b81b80 100644 --- a/selfdrive/manager/process_config.py +++ b/selfdrive/manager/process_config.py @@ -76,10 +76,10 @@ procs = [ PythonProcess("plannerd", "selfdrive.controls.plannerd", only_onroad), PythonProcess("radard", "selfdrive.controls.radard", only_onroad), PythonProcess("thermald", "selfdrive.thermald.thermald", always_run), - PythonProcess("tombstoned", "selfdrive.tombstoned", always_run, enabled=not PC), + PythonProcess("tombstoned", "system.tombstoned", always_run, enabled=not PC), PythonProcess("updated", "selfdrive.updated.updated", only_offroad, enabled=not PC), PythonProcess("uploader", "system.loggerd.uploader", always_run), - PythonProcess("statsd", "selfdrive.statsd", always_run), + PythonProcess("statsd", "system.statsd", always_run), # debug procs NativeProcess("bridge", "cereal/messaging", ["./bridge"], notcar), diff --git a/selfdrive/modeld/modeld.py b/selfdrive/modeld/modeld.py index 66563a3c1c..526fd4d218 100755 --- a/selfdrive/modeld/modeld.py +++ b/selfdrive/modeld/modeld.py @@ -15,7 +15,7 @@ from openpilot.common.filter_simple import FirstOrderFilter from openpilot.common.realtime import config_realtime_process from openpilot.common.transformations.camera import DEVICE_CAMERAS from openpilot.common.transformations.model import get_warp_matrix -from openpilot.selfdrive import sentry +from openpilot.system import sentry from openpilot.selfdrive.car.car_helpers import get_demo_car_params from openpilot.selfdrive.controls.lib.desire_helper import DesireHelper from openpilot.selfdrive.modeld.runners import ModelRunner, Runtime diff --git a/selfdrive/test/test_onroad.py b/selfdrive/test/test_onroad.py index 9aca9afcd9..53db737fc1 100755 --- a/selfdrive/test/test_onroad.py +++ b/selfdrive/test/test_onroad.py @@ -54,12 +54,12 @@ PROCS = { "selfdrive.monitoring.dmonitoringd": 4.0, "./proclogd": 1.54, "system.logmessaged": 0.2, - "selfdrive.tombstoned": 0, + "system.tombstoned": 0, "./logcatd": 0, "system.micd": 6.0, "system.timed": 0, "selfdrive.boardd.pandad": 0, - "selfdrive.statsd": 0.4, + "system.statsd": 0.4, "selfdrive.navd.navd": 0.4, "system.loggerd.uploader": (0.5, 15.0), "system.loggerd.deleter": 0.1, diff --git a/selfdrive/thermald/power_monitoring.py b/selfdrive/thermald/power_monitoring.py index 073589edb7..5a94625b48 100644 --- a/selfdrive/thermald/power_monitoring.py +++ b/selfdrive/thermald/power_monitoring.py @@ -4,7 +4,7 @@ import threading from openpilot.common.params import Params from openpilot.system.hardware import HARDWARE from openpilot.common.swaglog import cloudlog -from openpilot.selfdrive.statsd import statlog +from openpilot.system.statsd import statlog CAR_VOLTAGE_LOW_PASS_K = 0.011 # LPF gain for 45s tau (dt/tau / (dt/tau + 1)) diff --git a/selfdrive/thermald/thermald.py b/selfdrive/thermald/thermald.py index f5fc949637..d44ce6c18d 100755 --- a/selfdrive/thermald/thermald.py +++ b/selfdrive/thermald/thermald.py @@ -19,7 +19,7 @@ from openpilot.common.realtime import DT_TRML from openpilot.selfdrive.controls.lib.alertmanager import set_offroad_alert from openpilot.system.hardware import HARDWARE, TICI, AGNOS from openpilot.system.loggerd.config import get_available_percent -from openpilot.selfdrive.statsd import statlog +from openpilot.system.statsd import statlog from openpilot.common.swaglog import cloudlog from openpilot.selfdrive.thermald.power_monitoring import PowerMonitoring from openpilot.selfdrive.thermald.fan_controller import TiciFanController diff --git a/selfdrive/sentry.py b/system/sentry.py similarity index 100% rename from selfdrive/sentry.py rename to system/sentry.py diff --git a/selfdrive/statsd.py b/system/statsd.py similarity index 100% rename from selfdrive/statsd.py rename to system/statsd.py diff --git a/selfdrive/tombstoned.py b/system/tombstoned.py similarity index 99% rename from selfdrive/tombstoned.py rename to system/tombstoned.py index 2c99c7eafe..5bcced2666 100755 --- a/selfdrive/tombstoned.py +++ b/system/tombstoned.py @@ -9,7 +9,7 @@ import time import glob from typing import NoReturn -import openpilot.selfdrive.sentry as sentry +import openpilot.system.sentry as sentry from openpilot.system.hardware.hw import Paths from openpilot.common.swaglog import cloudlog from openpilot.system.version import get_build_metadata From b2cf9b35f68061587fd6ab7a94a58f7b433b2da2 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Mon, 20 May 2024 22:51:29 -0700 Subject: [PATCH 15/26] thermald: move to system/ (#32494) * thermald: move to system/ * fix path * revert --- pyproject.toml | 2 +- release/files_common | 6 +++--- selfdrive/manager/process_config.py | 2 +- selfdrive/test/test_onroad.py | 2 +- {selfdrive => system}/thermald/__init__.py | 0 {selfdrive => system}/thermald/fan_controller.py | 0 {selfdrive => system}/thermald/power_monitoring.py | 0 {selfdrive => system}/thermald/tests/__init__.py | 0 {selfdrive => system}/thermald/tests/test_fan_controller.py | 2 +- .../thermald/tests/test_power_monitoring.py | 6 +++--- {selfdrive => system}/thermald/thermald.py | 4 ++-- 11 files changed, 12 insertions(+), 12 deletions(-) rename {selfdrive => system}/thermald/__init__.py (100%) rename {selfdrive => system}/thermald/fan_controller.py (100%) rename {selfdrive => system}/thermald/power_monitoring.py (100%) rename {selfdrive => system}/thermald/tests/__init__.py (100%) rename {selfdrive => system}/thermald/tests/test_fan_controller.py (96%) rename {selfdrive => system}/thermald/tests/test_power_monitoring.py (96%) rename {selfdrive => system}/thermald/thermald.py (99%) diff --git a/pyproject.toml b/pyproject.toml index 2b183c983e..db1b4195a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,10 +26,10 @@ testpaths = [ "selfdrive/locationd", "selfdrive/monitoring", "selfdrive/navd/tests", - "selfdrive/thermald", "selfdrive/test/longitudinal_maneuvers", "selfdrive/test/process_replay/test_fuzzy.py", "selfdrive/updated", + "system/thermald", "system/athena", "system/camerad", "system/hardware/tici", diff --git a/release/files_common b/release/files_common index 19a93bc7da..8636959da5 100644 --- a/release/files_common +++ b/release/files_common @@ -253,9 +253,9 @@ system/webrtc/schema.py system/webrtc/device/audio.py system/webrtc/device/video.py -selfdrive/thermald/thermald.py -selfdrive/thermald/power_monitoring.py -selfdrive/thermald/fan_controller.py +system/thermald/thermald.py +system/thermald/power_monitoring.py +system/thermald/fan_controller.py selfdrive/test/__init__.py selfdrive/test/fuzzy_generation.py diff --git a/selfdrive/manager/process_config.py b/selfdrive/manager/process_config.py index 5ea4b81b80..c2aa3b44e2 100644 --- a/selfdrive/manager/process_config.py +++ b/selfdrive/manager/process_config.py @@ -75,7 +75,7 @@ procs = [ PythonProcess("pigeond", "system.ubloxd.pigeond", ublox, enabled=TICI), PythonProcess("plannerd", "selfdrive.controls.plannerd", only_onroad), PythonProcess("radard", "selfdrive.controls.radard", only_onroad), - PythonProcess("thermald", "selfdrive.thermald.thermald", always_run), + PythonProcess("thermald", "system.thermald.thermald", always_run), PythonProcess("tombstoned", "system.tombstoned", always_run, enabled=not PC), PythonProcess("updated", "selfdrive.updated.updated", only_offroad, enabled=not PC), PythonProcess("uploader", "system.loggerd.uploader", always_run), diff --git a/selfdrive/test/test_onroad.py b/selfdrive/test/test_onroad.py index 53db737fc1..14f7773302 100755 --- a/selfdrive/test/test_onroad.py +++ b/selfdrive/test/test_onroad.py @@ -47,7 +47,7 @@ PROCS = { "selfdrive.controls.radard": 7.0, "selfdrive.modeld.modeld": 13.0, "selfdrive.modeld.dmonitoringmodeld": 8.0, - "selfdrive.thermald.thermald": 3.87, + "system.thermald.thermald": 3.87, "selfdrive.locationd.calibrationd": 2.0, "selfdrive.locationd.torqued": 5.0, "selfdrive.ui.soundd": 3.5, diff --git a/selfdrive/thermald/__init__.py b/system/thermald/__init__.py similarity index 100% rename from selfdrive/thermald/__init__.py rename to system/thermald/__init__.py diff --git a/selfdrive/thermald/fan_controller.py b/system/thermald/fan_controller.py similarity index 100% rename from selfdrive/thermald/fan_controller.py rename to system/thermald/fan_controller.py diff --git a/selfdrive/thermald/power_monitoring.py b/system/thermald/power_monitoring.py similarity index 100% rename from selfdrive/thermald/power_monitoring.py rename to system/thermald/power_monitoring.py diff --git a/selfdrive/thermald/tests/__init__.py b/system/thermald/tests/__init__.py similarity index 100% rename from selfdrive/thermald/tests/__init__.py rename to system/thermald/tests/__init__.py diff --git a/selfdrive/thermald/tests/test_fan_controller.py b/system/thermald/tests/test_fan_controller.py similarity index 96% rename from selfdrive/thermald/tests/test_fan_controller.py rename to system/thermald/tests/test_fan_controller.py index c2b1a64509..5c858132a2 100755 --- a/selfdrive/thermald/tests/test_fan_controller.py +++ b/system/thermald/tests/test_fan_controller.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import pytest -from openpilot.selfdrive.thermald.fan_controller import TiciFanController +from openpilot.system.thermald.fan_controller import TiciFanController ALL_CONTROLLERS = [TiciFanController] diff --git a/selfdrive/thermald/tests/test_power_monitoring.py b/system/thermald/tests/test_power_monitoring.py similarity index 96% rename from selfdrive/thermald/tests/test_power_monitoring.py rename to system/thermald/tests/test_power_monitoring.py index f68191475b..0795a29c8f 100755 --- a/selfdrive/thermald/tests/test_power_monitoring.py +++ b/system/thermald/tests/test_power_monitoring.py @@ -2,7 +2,7 @@ import pytest from openpilot.common.params import Params -from openpilot.selfdrive.thermald.power_monitoring import PowerMonitoring, CAR_BATTERY_CAPACITY_uWh, \ +from openpilot.system.thermald.power_monitoring import PowerMonitoring, CAR_BATTERY_CAPACITY_uWh, \ CAR_CHARGING_RATE_W, VBATT_PAUSE_CHARGING, DELAY_SHUTDOWN_TIME_S # Create fake time @@ -18,9 +18,9 @@ VOLTAGE_BELOW_PAUSE_CHARGING = (VBATT_PAUSE_CHARGING - 1) * 1e3 def pm_patch(mocker, name, value, constant=False): if constant: - mocker.patch(f"openpilot.selfdrive.thermald.power_monitoring.{name}", value) + mocker.patch(f"openpilot.system.thermald.power_monitoring.{name}", value) else: - mocker.patch(f"openpilot.selfdrive.thermald.power_monitoring.{name}", return_value=value) + mocker.patch(f"openpilot.system.thermald.power_monitoring.{name}", return_value=value) @pytest.fixture(autouse=True) diff --git a/selfdrive/thermald/thermald.py b/system/thermald/thermald.py similarity index 99% rename from selfdrive/thermald/thermald.py rename to system/thermald/thermald.py index d44ce6c18d..90f6494a26 100755 --- a/selfdrive/thermald/thermald.py +++ b/system/thermald/thermald.py @@ -21,8 +21,8 @@ from openpilot.system.hardware import HARDWARE, TICI, AGNOS from openpilot.system.loggerd.config import get_available_percent from openpilot.system.statsd import statlog from openpilot.common.swaglog import cloudlog -from openpilot.selfdrive.thermald.power_monitoring import PowerMonitoring -from openpilot.selfdrive.thermald.fan_controller import TiciFanController +from openpilot.system.thermald.power_monitoring import PowerMonitoring +from openpilot.system.thermald.fan_controller import TiciFanController from openpilot.system.version import terms_version, training_version ThermalStatus = log.DeviceState.ThermalStatus From 936e8d3d801e3ef52660eb0940293197598514bf Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Mon, 20 May 2024 23:01:42 -0700 Subject: [PATCH 16/26] CI: merge build jobs (#32495) --- .github/workflows/selfdrive_tests.yaml | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index d3c8009bc2..1c83c39bbb 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -76,24 +76,8 @@ jobs: - uses: actions/checkout@v4 with: submodules: true - - uses: ./.github/workflows/setup-with-retry - with: - docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }} - - uses: ./.github/workflows/compile-openpilot - timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache - - docker_push: - name: docker push - strategy: - matrix: - arch: ${{ fromJson( (github.repository == 'commaai/openpilot') && '["x86_64", "aarch64"]' || '["x86_64"]' ) }} - runs-on: ${{ (matrix.arch == 'aarch64') && 'namespace-profile-arm64-2x8' || 'ubuntu-latest' }} - if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Setup to push to repo + - name: Setup docker push + if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' run: | echo "PUSH_IMAGE=true" >> "$GITHUB_ENV" echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV" @@ -101,12 +85,14 @@ jobs: - uses: ./.github/workflows/setup-with-retry with: docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }} + - uses: ./.github/workflows/compile-openpilot + timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache docker_push_multiarch: name: docker push multiarch tag runs-on: ubuntu-latest if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' - needs: [docker_push] + needs: [build] steps: - uses: actions/checkout@v4 with: From da42c4a561343749a4970d7908ed7a3b2ed0402e Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Mon, 20 May 2024 23:11:19 -0700 Subject: [PATCH 17/26] CI: move car docs diff job (#32496) * CI: move car docs diff job * no if --- .github/workflows/auto_pr_review.yaml | 54 +++++++++++++++++++++++++- .github/workflows/selfdrive_tests.yaml | 54 -------------------------- 2 files changed, 53 insertions(+), 55 deletions(-) diff --git a/.github/workflows/auto_pr_review.yaml b/.github/workflows/auto_pr_review.yaml index 8316fedda0..2c41dc8bdb 100644 --- a/.github/workflows/auto_pr_review.yaml +++ b/.github/workflows/auto_pr_review.yaml @@ -54,6 +54,59 @@ jobs: comment_tag: run_id GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + car_docs_diff: + name: PR comments + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + ref: ${{ github.event.pull_request.base.ref }} + - run: git lfs pull + - uses: ./.github/workflows/setup-with-retry + - name: Get base car info + run: | + ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_docs.py --path /tmp/openpilot_cache/base_car_docs" + sudo chown -R $USER:$USER ${{ github.workspace }} + - uses: actions/checkout@v4 + with: + submodules: true + path: current + - run: cd current && git lfs pull + - name: Save car docs diff + id: save_diff + run: | + cd current + ${{ env.RUN }} "scons -j$(nproc)" + output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_docs") + output="${output//$'\n'/'%0A'}" + echo "::set-output name=diff::$output" + - name: Find comment + if: ${{ env.AZURE_TOKEN != '' }} + uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + body-includes: This PR makes changes to + - name: Update comment + if: ${{ steps.save_diff.outputs.diff != '' && env.AZURE_TOKEN != '' }} + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body: "${{ steps.save_diff.outputs.diff }}" + edit-mode: replace + - name: Delete comment + if: ${{ steps.fc.outputs.comment-id != '' && steps.save_diff.outputs.diff == '' && env.AZURE_TOKEN != '' }} + uses: actions/github-script@v7 + with: + script: | + github.rest.issues.deleteComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: ${{ steps.fc.outputs.comment-id }} + }) + check-pr-template: runs-on: ubuntu-latest permissions: @@ -224,4 +277,3 @@ jobs: console.log("Label 'in-bot-review' not found, ignoring"); }); } - diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 1c83c39bbb..2068eeb012 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -249,60 +249,6 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - car_docs_diff: - name: PR comments - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' - steps: - - uses: actions/checkout@v4 - with: - submodules: true - ref: ${{ github.event.pull_request.base.ref }} - - run: git lfs pull - - uses: ./.github/workflows/setup-with-retry - - name: Get base car info - run: | - ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_docs.py --path /tmp/openpilot_cache/base_car_docs" - sudo chown -R $USER:$USER ${{ github.workspace }} - - uses: actions/checkout@v4 - with: - submodules: true - path: current - - run: cd current && git lfs pull - - name: Save car docs diff - id: save_diff - run: | - cd current - ${{ env.RUN }} "scons -j$(nproc)" - output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_docs") - output="${output//$'\n'/'%0A'}" - echo "::set-output name=diff::$output" - - name: Find comment - if: ${{ env.AZURE_TOKEN != '' }} - uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e - id: fc - with: - issue-number: ${{ github.event.pull_request.number }} - body-includes: This PR makes changes to - - name: Update comment - if: ${{ steps.save_diff.outputs.diff != '' && env.AZURE_TOKEN != '' }} - uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 - with: - comment-id: ${{ steps.fc.outputs.comment-id }} - issue-number: ${{ github.event.pull_request.number }} - body: "${{ steps.save_diff.outputs.diff }}" - edit-mode: replace - - name: Delete comment - if: ${{ steps.fc.outputs.comment-id != '' && steps.save_diff.outputs.diff == '' && env.AZURE_TOKEN != '' }} - uses: actions/github-script@v7 - with: - script: | - github.rest.issues.deleteComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: ${{ steps.fc.outputs.comment-id }} - }) - create_ui_report: name: Create UI Report runs-on: ubuntu-latest From ce136317d8e7c9d1c2981dacc77df414859c0e21 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 21 May 2024 01:26:47 -0500 Subject: [PATCH 18/26] regen: use existing carParams msg (#32493) use existing msg --- selfdrive/test/process_replay/migration.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/selfdrive/test/process_replay/migration.py b/selfdrive/test/process_replay/migration.py index 152f281948..be78c92d73 100644 --- a/selfdrive/test/process_replay/migration.py +++ b/selfdrive/test/process_replay/migration.py @@ -183,9 +183,7 @@ def migrate_carParams(lr, old_logtime=False): all_msgs = [] for msg in lr: if msg.which() == 'carParams': - CP = messaging.new_message('carParams') - CP.valid = True - CP.carParams = msg.carParams.as_builder() + CP = msg.as_builder() CP.carParams.carFingerprint = MIGRATION.get(CP.carParams.carFingerprint, CP.carParams.carFingerprint) for car_fw in CP.carParams.carFw: car_fw.brand = CP.carParams.carName From 5e98d9e2895330264a42c90c81c347b7d5bd1f5c Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Mon, 20 May 2024 23:19:09 -0700 Subject: [PATCH 19/26] also tag as latest --- selfdrive/test/docker_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfdrive/test/docker_build.sh b/selfdrive/test/docker_build.sh index 5f77ceb10d..4d58a1507c 100755 --- a/selfdrive/test/docker_build.sh +++ b/selfdrive/test/docker_build.sh @@ -17,7 +17,7 @@ fi source $SCRIPT_DIR/docker_common.sh $1 "$TAG_SUFFIX" -DOCKER_BUILDKIT=1 docker buildx build --provenance false --pull --platform $PLATFORM --load --cache-to type=inline --cache-from type=registry,ref=$REMOTE_TAG -t $REMOTE_TAG -t $LOCAL_TAG -f $OPENPILOT_DIR/$DOCKER_FILE $OPENPILOT_DIR +DOCKER_BUILDKIT=1 docker buildx build --provenance false --pull --platform $PLATFORM --load --cache-to type=inline --cache-from type=registry,ref=$REMOTE_TAG -t $DOCKER_IMAGE:latest -t $REMOTE_TAG -t $LOCAL_TAG -f $OPENPILOT_DIR/$DOCKER_FILE $OPENPILOT_DIR if [ -n "$PUSH_IMAGE" ]; then docker push $REMOTE_TAG From 49d7edfe11add9abc09b46b271ea8b646797c080 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Mon, 20 May 2024 23:33:16 -0700 Subject: [PATCH 20/26] Revert "CI: move car docs diff job (#32496)" This reverts commit da42c4a561343749a4970d7908ed7a3b2ed0402e. --- .github/workflows/auto_pr_review.yaml | 54 +------------------------- .github/workflows/selfdrive_tests.yaml | 54 ++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 53 deletions(-) diff --git a/.github/workflows/auto_pr_review.yaml b/.github/workflows/auto_pr_review.yaml index 2c41dc8bdb..8316fedda0 100644 --- a/.github/workflows/auto_pr_review.yaml +++ b/.github/workflows/auto_pr_review.yaml @@ -54,59 +54,6 @@ jobs: comment_tag: run_id GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - car_docs_diff: - name: PR comments - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - ref: ${{ github.event.pull_request.base.ref }} - - run: git lfs pull - - uses: ./.github/workflows/setup-with-retry - - name: Get base car info - run: | - ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_docs.py --path /tmp/openpilot_cache/base_car_docs" - sudo chown -R $USER:$USER ${{ github.workspace }} - - uses: actions/checkout@v4 - with: - submodules: true - path: current - - run: cd current && git lfs pull - - name: Save car docs diff - id: save_diff - run: | - cd current - ${{ env.RUN }} "scons -j$(nproc)" - output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_docs") - output="${output//$'\n'/'%0A'}" - echo "::set-output name=diff::$output" - - name: Find comment - if: ${{ env.AZURE_TOKEN != '' }} - uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e - id: fc - with: - issue-number: ${{ github.event.pull_request.number }} - body-includes: This PR makes changes to - - name: Update comment - if: ${{ steps.save_diff.outputs.diff != '' && env.AZURE_TOKEN != '' }} - uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 - with: - comment-id: ${{ steps.fc.outputs.comment-id }} - issue-number: ${{ github.event.pull_request.number }} - body: "${{ steps.save_diff.outputs.diff }}" - edit-mode: replace - - name: Delete comment - if: ${{ steps.fc.outputs.comment-id != '' && steps.save_diff.outputs.diff == '' && env.AZURE_TOKEN != '' }} - uses: actions/github-script@v7 - with: - script: | - github.rest.issues.deleteComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: ${{ steps.fc.outputs.comment-id }} - }) - check-pr-template: runs-on: ubuntu-latest permissions: @@ -277,3 +224,4 @@ jobs: console.log("Label 'in-bot-review' not found, ignoring"); }); } + diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 2068eeb012..1c83c39bbb 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -249,6 +249,60 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + car_docs_diff: + name: PR comments + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - uses: actions/checkout@v4 + with: + submodules: true + ref: ${{ github.event.pull_request.base.ref }} + - run: git lfs pull + - uses: ./.github/workflows/setup-with-retry + - name: Get base car info + run: | + ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_docs.py --path /tmp/openpilot_cache/base_car_docs" + sudo chown -R $USER:$USER ${{ github.workspace }} + - uses: actions/checkout@v4 + with: + submodules: true + path: current + - run: cd current && git lfs pull + - name: Save car docs diff + id: save_diff + run: | + cd current + ${{ env.RUN }} "scons -j$(nproc)" + output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_docs") + output="${output//$'\n'/'%0A'}" + echo "::set-output name=diff::$output" + - name: Find comment + if: ${{ env.AZURE_TOKEN != '' }} + uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + body-includes: This PR makes changes to + - name: Update comment + if: ${{ steps.save_diff.outputs.diff != '' && env.AZURE_TOKEN != '' }} + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body: "${{ steps.save_diff.outputs.diff }}" + edit-mode: replace + - name: Delete comment + if: ${{ steps.fc.outputs.comment-id != '' && steps.save_diff.outputs.diff == '' && env.AZURE_TOKEN != '' }} + uses: actions/github-script@v7 + with: + script: | + github.rest.issues.deleteComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: ${{ steps.fc.outputs.comment-id }} + }) + create_ui_report: name: Create UI Report runs-on: ubuntu-latest From 71f5c441fe32184d94a9f26565a36c661e2ccf28 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 21 May 2024 03:18:10 -0500 Subject: [PATCH 21/26] card: process that abstracts car interface and CAN (#32380) * format card * standalone process * no class member CS, there's no point also can be confusing; what else could be using this? * rename CoS * Update selfdrive/controls/controlsd.py * never works first time :D * canRcvTimeout is bool * hack * add cpu * see what testing closet comes up with * first * some clean up * support passable CI, fix test models * fix startup alert * process replay changes * test_fuzzy * gate carOutput valid on carControl valid * we should publish after we update carOutput * controlsd was using actuatorsOutput from 2 frames ago for torque, not the most up to date * check all checks for carControl in case controlsd dies * log more timestamps * more generic latency logger; needs some clean up latency_logger.py was difficult to understand and modify * card polls on can and carControl to get latest carControl possible * temp try to send earlier * add log * remove latencylogger * no mpld3! * old loop * detect first event * normal send * revert "card polls on can and carControl to get latest carControl possible" how it was is best * sheesh! update should be first * first timestamp * temp comment ( timestamp is slow :( ) * more final ordering, and make polling on/off test repeatable * Received can * new plot timestamps * clean up * no poll * add controllers (draft) * Revert "add controllers (draft)" This reverts commit e2c3f01b2fadcff74347bac90c8a5cc1ef4e27b3. * fix that * conventions * just use CS * consider controlsd state machine in card: not fully done * hmm it's just becoming controlsd * rm debugging * Revert "hmm it's just becoming controlsd" This reverts commit 534a357ee95bec4ed070667186af55d59421bbc7. * Revert "just use CS" This reverts commit 9fa7406f30c86200f20457f7b9ff95e731201bf9. * add vCruise * migrate car state * Revert "migrate car state" This reverts commit 4ae86ca163c6920070f410f608f7644ab632850b. * Revert "add vCruise" This reverts commit af247a8da41c3626ada4231b98042da1a1ae4633. * simple state machine in card (doesn't work as is) * Revert "simple state machine in card (doesn't work as is)" This reverts commit b4af8a9b0a2e17fdfc89d344c64678ef51305c24. * poll carState without conflate * bump * remove state transition * fix * update refs * ignore cumLagMs and don't ignore valid * fix controls mismatch; controlsd used to set alt exp * controlsd_config_callback not needed for card * revert ref temp * update refs * no poll * not builder! * test fix * need to migrate initialized * CC will be a reader * more as_reader! * fix None * init after publish like before - no real difference * controlsd clean up * remove redundant check and check passive for init * stash * flip * migrate missing carOutput for controlsd * Update ref_commit * bump cereal * comment * no class params * no class * Revert "no class" This reverts commit 5499b83c2dcb5462070626f8523e3aec6f4c209d. * add todo * regen and update refs * fix * update refs * and fix that * should be controlsstate * remove controlsState migration CoS.initialized isn't needed yet * fix * flip! * bump * fix that * update refs * fix * if canValid goes false, controlsd would still send * bump * rm diff * need to be very careful with initializing * update refs --- cereal | 2 +- selfdrive/car/body/carcontroller.py | 2 +- selfdrive/car/card.py | 104 ++++++++++++------ selfdrive/car/chrysler/carcontroller.py | 2 +- selfdrive/car/ford/carcontroller.py | 2 +- selfdrive/car/gm/carcontroller.py | 2 +- selfdrive/car/honda/carcontroller.py | 2 +- selfdrive/car/hyundai/carcontroller.py | 2 +- selfdrive/car/mazda/carcontroller.py | 2 +- selfdrive/car/mock/carcontroller.py | 2 +- selfdrive/car/nissan/carcontroller.py | 2 +- selfdrive/car/subaru/carcontroller.py | 2 +- selfdrive/car/tesla/carcontroller.py | 2 +- selfdrive/car/tests/test_car_interfaces.py | 4 +- selfdrive/car/tests/test_models.py | 12 +- selfdrive/car/toyota/carcontroller.py | 2 +- selfdrive/car/volkswagen/carcontroller.py | 2 +- selfdrive/controls/controlsd.py | 42 +++---- selfdrive/controls/tests/test_startup.py | 3 +- selfdrive/manager/process_config.py | 1 + selfdrive/test/process_replay/migration.py | 18 +++ .../test/process_replay/process_replay.py | 32 ++++-- selfdrive/test/process_replay/ref_commit | 2 +- selfdrive/test/process_replay/test_fuzzy.py | 2 +- .../test/process_replay/test_processes.py | 36 +++--- selfdrive/test/test_onroad.py | 3 +- 26 files changed, 181 insertions(+), 106 deletions(-) diff --git a/cereal b/cereal index 0a9b426e55..5812f2c075 160000 --- a/cereal +++ b/cereal @@ -1 +1 @@ -Subproject commit 0a9b426e55653daea6cc9d3c40c3f7600ec0db49 +Subproject commit 5812f2c075a5364cecafe4e8ed68a12b12a5631f diff --git a/selfdrive/car/body/carcontroller.py b/selfdrive/car/body/carcontroller.py index db34320caa..259126c416 100644 --- a/selfdrive/car/body/carcontroller.py +++ b/selfdrive/car/body/carcontroller.py @@ -75,7 +75,7 @@ class CarController(CarControllerBase): can_sends = [] can_sends.append(bodycan.create_control(self.packer, torque_l, torque_r)) - new_actuators = CC.actuators.copy() + new_actuators = CC.actuators.as_builder() new_actuators.accel = torque_l new_actuators.steer = torque_r new_actuators.steerOutputCan = torque_r diff --git a/selfdrive/car/card.py b/selfdrive/car/card.py index ed916a6fe1..2705a3c01c 100755 --- a/selfdrive/car/card.py +++ b/selfdrive/car/card.py @@ -9,7 +9,7 @@ from cereal import car from panda import ALTERNATIVE_EXPERIENCE from openpilot.common.params import Params -from openpilot.common.realtime import DT_CTRL +from openpilot.common.realtime import config_realtime_process, Priority, Ratekeeper, DT_CTRL from openpilot.selfdrive.boardd.boardd import can_list_to_can_capnp from openpilot.selfdrive.car.car_helpers import get_car, get_one_can @@ -21,22 +21,24 @@ REPLAY = "REPLAY" in os.environ EventName = car.CarEvent.EventName -class CarD: +class Car: CI: CarInterfaceBase def __init__(self, CI=None): self.can_sock = messaging.sub_sock('can', timeout=20) - self.sm = messaging.SubMaster(['pandaStates']) + self.sm = messaging.SubMaster(['pandaStates', 'carControl', 'onroadEvents']) self.pm = messaging.PubMaster(['sendcan', 'carState', 'carParams', 'carOutput']) self.can_rcv_timeout_counter = 0 # consecutive timeout count self.can_rcv_cum_timeout_counter = 0 # cumulative timeout count self.CC_prev = car.CarControl.new_message() + self.CS_prev = car.CarState.new_message() + self.initialized_prev = False - self.last_actuators = None + self.last_actuators_output = car.CarControl.Actuators.new_message() - self.params = Params() + params = Params() if CI is None: # wait for one pandaState and one CAN packet @@ -44,18 +46,18 @@ class CarD: get_one_can(self.can_sock) num_pandas = len(messaging.recv_one_retry(self.sm.sock['pandaStates']).pandaStates) - experimental_long_allowed = self.params.get_bool("ExperimentalLongitudinalEnabled") + experimental_long_allowed = params.get_bool("ExperimentalLongitudinalEnabled") self.CI, self.CP = get_car(self.can_sock, self.pm.sock['sendcan'], experimental_long_allowed, num_pandas) else: self.CI, self.CP = CI, CI.CP # set alternative experiences from parameters - self.disengage_on_accelerator = self.params.get_bool("DisengageOnAccelerator") + self.disengage_on_accelerator = params.get_bool("DisengageOnAccelerator") self.CP.alternativeExperience = 0 if not self.disengage_on_accelerator: self.CP.alternativeExperience |= ALTERNATIVE_EXPERIENCE.DISABLE_DISENGAGE_ON_GAS - openpilot_enabled_toggle = self.params.get_bool("OpenpilotEnabledToggle") + openpilot_enabled_toggle = params.get_bool("OpenpilotEnabledToggle") controller_available = self.CI.CC is not None and openpilot_enabled_toggle and not self.CP.dashcamOnly @@ -66,22 +68,20 @@ class CarD: self.CP.safetyConfigs = [safety_config] # Write previous route's CarParams - prev_cp = self.params.get("CarParamsPersistent") + prev_cp = params.get("CarParamsPersistent") if prev_cp is not None: - self.params.put("CarParamsPrevRoute", prev_cp) + params.put("CarParamsPrevRoute", prev_cp) # Write CarParams for controls and radard cp_bytes = self.CP.to_bytes() - self.params.put("CarParams", cp_bytes) - self.params.put_nonblocking("CarParamsCache", cp_bytes) - self.params.put_nonblocking("CarParamsPersistent", cp_bytes) + params.put("CarParams", cp_bytes) + params.put_nonblocking("CarParamsCache", cp_bytes) + params.put_nonblocking("CarParamsPersistent", cp_bytes) - self.CS_prev = car.CarState.new_message() self.events = Events() - def initialize(self): - """Initialize CarInterface, once controls are ready""" - self.CI.init(self.CP, self.can_sock, self.pm.sock['sendcan']) + # card is driven by can recv, expected at 100Hz + self.rk = Ratekeeper(100, print_delay_threshold=None) def state_update(self) -> car.CarState: """carState update loop, driven by can""" @@ -106,11 +106,6 @@ class CarD: if can_rcv_valid and REPLAY: self.can_log_mono_time = messaging.log_from_bytes(can_strs[0]).logMonoTime - self.update_events(CS) - self.state_publish(CS) - - CS = CS.as_reader() - self.CS_prev = CS return CS def update_events(self, CS: car.CarState) -> car.CarState: @@ -129,12 +124,6 @@ class CarD: def state_publish(self, CS: car.CarState): """carState and carParams publish loop""" - # carState - cs_send = messaging.new_message('carState') - cs_send.valid = CS.canValid - cs_send.carState = CS - self.pm.send('carState', cs_send) - # carParams - logged every 50 seconds (> 1 per segment) if self.sm.frame % int(50. / DT_CTRL) == 0: cp_send = messaging.new_message('carParams') @@ -144,17 +133,60 @@ class CarD: # publish new carOutput co_send = messaging.new_message('carOutput') - co_send.valid = True - if self.last_actuators is not None: - co_send.carOutput.actuatorsOutput = self.last_actuators + co_send.valid = self.sm.all_checks(['carControl']) + co_send.carOutput.actuatorsOutput = self.last_actuators_output self.pm.send('carOutput', co_send) + # kick off controlsd step while we actuate the latest carControl packet + cs_send = messaging.new_message('carState') + cs_send.valid = CS.canValid + cs_send.carState = CS + cs_send.carState.canRcvTimeout = self.can_rcv_timeout + cs_send.carState.canErrorCounter = self.can_rcv_cum_timeout_counter + cs_send.carState.cumLagMs = -self.rk.remaining * 1000. + self.pm.send('carState', cs_send) + def controls_update(self, CS: car.CarState, CC: car.CarControl): """control update loop, driven by carControl""" - # send car controls over can - now_nanos = self.can_log_mono_time if REPLAY else int(time.monotonic() * 1e9) - self.last_actuators, can_sends = self.CI.apply(CC, now_nanos) - self.pm.send('sendcan', can_list_to_can_capnp(can_sends, msgtype='sendcan', valid=CS.canValid)) + if not self.initialized_prev: + # Initialize CarInterface, once controls are ready + self.CI.init(self.CP, self.can_sock, self.pm.sock['sendcan']) + + if self.sm.all_alive(['carControl']): + # send car controls over can + now_nanos = self.can_log_mono_time if REPLAY else int(time.monotonic() * 1e9) + self.last_actuators_output, can_sends = self.CI.apply(CC, now_nanos) + self.pm.send('sendcan', can_list_to_can_capnp(can_sends, msgtype='sendcan', valid=CS.canValid)) + + self.CC_prev = CC + + def step(self): + CS = self.state_update() + + self.update_events(CS) + + self.state_publish(CS) + + initialized = (not any(e.name == EventName.controlsInitializing for e in self.sm['onroadEvents']) and + self.sm.seen['onroadEvents']) + if not self.CP.passive and initialized: + self.controls_update(CS, self.sm['carControl']) + + self.initialized_prev = initialized + self.CS_prev = CS.as_reader() + + def card_thread(self): + while True: + self.step() + self.rk.monitor_time() + + +def main(): + config_realtime_process(4, Priority.CTRL_HIGH) + car = Car() + car.card_thread() + - self.CC_prev = CC +if __name__ == "__main__": + main() diff --git a/selfdrive/car/chrysler/carcontroller.py b/selfdrive/car/chrysler/carcontroller.py index 39248f3f75..85f53f68eb 100644 --- a/selfdrive/car/chrysler/carcontroller.py +++ b/selfdrive/car/chrysler/carcontroller.py @@ -78,7 +78,7 @@ class CarController(CarControllerBase): self.frame += 1 - new_actuators = CC.actuators.copy() + new_actuators = CC.actuators.as_builder() new_actuators.steer = self.apply_steer_last / self.params.STEER_MAX new_actuators.steerOutputCan = self.apply_steer_last diff --git a/selfdrive/car/ford/carcontroller.py b/selfdrive/car/ford/carcontroller.py index 47082fb56f..7be3b2ebe9 100644 --- a/selfdrive/car/ford/carcontroller.py +++ b/selfdrive/car/ford/carcontroller.py @@ -113,7 +113,7 @@ class CarController(CarControllerBase): self.steer_alert_last = steer_alert self.lead_distance_bars_last = hud_control.leadDistanceBars - new_actuators = actuators.copy() + new_actuators = actuators.as_builder() new_actuators.curvature = self.apply_curvature_last self.frame += 1 diff --git a/selfdrive/car/gm/carcontroller.py b/selfdrive/car/gm/carcontroller.py index f8d747029b..b204d3b80f 100644 --- a/selfdrive/car/gm/carcontroller.py +++ b/selfdrive/car/gm/carcontroller.py @@ -155,7 +155,7 @@ class CarController(CarControllerBase): if self.frame % 10 == 0: can_sends.append(gmcan.create_pscm_status(self.packer_pt, CanBus.CAMERA, CS.pscm_status)) - new_actuators = actuators.copy() + new_actuators = actuators.as_builder() new_actuators.steer = self.apply_steer_last / self.params.STEER_MAX new_actuators.steerOutputCan = self.apply_steer_last new_actuators.gas = self.apply_gas diff --git a/selfdrive/car/honda/carcontroller.py b/selfdrive/car/honda/carcontroller.py index 6fe8c27585..fe023ea17d 100644 --- a/selfdrive/car/honda/carcontroller.py +++ b/selfdrive/car/honda/carcontroller.py @@ -244,7 +244,7 @@ class CarController(CarControllerBase): self.speed = pcm_speed self.gas = pcm_accel / self.params.NIDEC_GAS_MAX - new_actuators = actuators.copy() + new_actuators = actuators.as_builder() new_actuators.speed = self.speed new_actuators.accel = self.accel new_actuators.gas = self.gas diff --git a/selfdrive/car/hyundai/carcontroller.py b/selfdrive/car/hyundai/carcontroller.py index 7829d764b0..4038ddcca9 100644 --- a/selfdrive/car/hyundai/carcontroller.py +++ b/selfdrive/car/hyundai/carcontroller.py @@ -163,7 +163,7 @@ class CarController(CarControllerBase): if self.frame % 50 == 0 and self.CP.openpilotLongitudinalControl: can_sends.append(hyundaican.create_frt_radar_opt(self.packer)) - new_actuators = actuators.copy() + new_actuators = actuators.as_builder() new_actuators.steer = apply_steer / self.params.STEER_MAX new_actuators.steerOutputCan = apply_steer new_actuators.accel = accel diff --git a/selfdrive/car/mazda/carcontroller.py b/selfdrive/car/mazda/carcontroller.py index 8d3a59b4ea..3d41634879 100644 --- a/selfdrive/car/mazda/carcontroller.py +++ b/selfdrive/car/mazda/carcontroller.py @@ -58,7 +58,7 @@ class CarController(CarControllerBase): can_sends.append(mazdacan.create_steering_control(self.packer, self.CP, self.frame, apply_steer, CS.cam_lkas)) - new_actuators = CC.actuators.copy() + new_actuators = CC.actuators.as_builder() new_actuators.steer = apply_steer / CarControllerParams.STEER_MAX new_actuators.steerOutputCan = apply_steer diff --git a/selfdrive/car/mock/carcontroller.py b/selfdrive/car/mock/carcontroller.py index 2b2da954ff..0cd37c0369 100644 --- a/selfdrive/car/mock/carcontroller.py +++ b/selfdrive/car/mock/carcontroller.py @@ -2,4 +2,4 @@ from openpilot.selfdrive.car.interfaces import CarControllerBase class CarController(CarControllerBase): def update(self, CC, CS, now_nanos): - return CC.actuators.copy(), [] + return CC.actuators.as_builder(), [] diff --git a/selfdrive/car/nissan/carcontroller.py b/selfdrive/car/nissan/carcontroller.py index 83775462b7..c7bd231398 100644 --- a/selfdrive/car/nissan/carcontroller.py +++ b/selfdrive/car/nissan/carcontroller.py @@ -75,7 +75,7 @@ class CarController(CarControllerBase): self.packer, CS.lkas_hud_info_msg, steer_hud_alert )) - new_actuators = actuators.copy() + new_actuators = actuators.as_builder() new_actuators.steeringAngleDeg = apply_angle self.frame += 1 diff --git a/selfdrive/car/subaru/carcontroller.py b/selfdrive/car/subaru/carcontroller.py index 22a1475b5b..d89ae8c639 100644 --- a/selfdrive/car/subaru/carcontroller.py +++ b/selfdrive/car/subaru/carcontroller.py @@ -136,7 +136,7 @@ class CarController(CarControllerBase): if self.frame % 2 == 0: can_sends.append(subarucan.create_es_static_2(self.packer)) - new_actuators = actuators.copy() + new_actuators = actuators.as_builder() new_actuators.steer = self.apply_steer_last / self.p.STEER_MAX new_actuators.steerOutputCan = self.apply_steer_last diff --git a/selfdrive/car/tesla/carcontroller.py b/selfdrive/car/tesla/carcontroller.py index f217c4692d..e460111e32 100644 --- a/selfdrive/car/tesla/carcontroller.py +++ b/selfdrive/car/tesla/carcontroller.py @@ -60,7 +60,7 @@ class CarController(CarControllerBase): # TODO: HUD control - new_actuators = actuators.copy() + new_actuators = actuators.as_builder() new_actuators.steeringAngleDeg = self.apply_angle_last self.frame += 1 diff --git a/selfdrive/car/tests/test_car_interfaces.py b/selfdrive/car/tests/test_car_interfaces.py index 4bbecd99fe..dfdf44215d 100755 --- a/selfdrive/car/tests/test_car_interfaces.py +++ b/selfdrive/car/tests/test_car_interfaces.py @@ -91,14 +91,14 @@ class TestCarInterfaces: CC = car.CarControl.new_message(**cc_msg) for _ in range(10): car_interface.update(CC, []) - car_interface.apply(CC, now_nanos) + car_interface.apply(CC.as_reader(), now_nanos) now_nanos += DT_CTRL * 1e9 # 10 ms CC = car.CarControl.new_message(**cc_msg) CC.enabled = True for _ in range(10): car_interface.update(CC, []) - car_interface.apply(CC, now_nanos) + car_interface.apply(CC.as_reader(), now_nanos) now_nanos += DT_CTRL * 1e9 # 10ms # Test controller initialization diff --git a/selfdrive/car/tests/test_models.py b/selfdrive/car/tests/test_models.py index 026693bdce..8807db69d9 100755 --- a/selfdrive/car/tests/test_models.py +++ b/selfdrive/car/tests/test_models.py @@ -15,7 +15,7 @@ from openpilot.common.basedir import BASEDIR from openpilot.common.params import Params from openpilot.common.realtime import DT_CTRL from openpilot.selfdrive.car import gen_empty_fingerprint -from openpilot.selfdrive.car.card import CarD +from openpilot.selfdrive.car.card import Car from openpilot.selfdrive.car.fingerprints import all_known_cars, MIGRATION from openpilot.selfdrive.car.car_helpers import FRAME_FINGERPRINT, interfaces from openpilot.selfdrive.car.honda.values import CAR as HONDA, HondaFlags @@ -215,7 +215,7 @@ class TestCarModelBase(unittest.TestCase): # TODO: also check for checksum violations from can parser can_invalid_cnt = 0 can_valid = False - CC = car.CarControl.new_message() + CC = car.CarControl.new_message().as_reader() for i, msg in enumerate(self.can_msgs): CS = self.CI.update(CC, (msg.as_builder().to_bytes(),)) @@ -308,17 +308,17 @@ class TestCarModelBase(unittest.TestCase): # Make sure we can send all messages while inactive CC = car.CarControl.new_message() - test_car_controller(CC) + test_car_controller(CC.as_reader()) # Test cancel + general messages (controls_allowed=False & cruise_engaged=True) self.safety.set_cruise_engaged_prev(True) CC = car.CarControl.new_message(cruiseControl={'cancel': True}) - test_car_controller(CC) + test_car_controller(CC.as_reader()) # Test resume + general messages (controls_allowed=True & cruise_engaged=True) self.safety.set_controls_allowed(True) CC = car.CarControl.new_message(cruiseControl={'resume': True}) - test_car_controller(CC) + test_car_controller(CC.as_reader()) # Skip stdout/stderr capture with pytest, causes elevated memory usage @pytest.mark.nocapture @@ -406,7 +406,7 @@ class TestCarModelBase(unittest.TestCase): controls_allowed_prev = False CS_prev = car.CarState.new_message() checks = defaultdict(int) - card = CarD(CI=self.CI) + card = Car(CI=self.CI) for idx, can in enumerate(self.can_msgs): CS = self.CI.update(CC, (can.as_builder().to_bytes(), )) for msg in filter(lambda m: m.src in range(64), can.can): diff --git a/selfdrive/car/toyota/carcontroller.py b/selfdrive/car/toyota/carcontroller.py index 354d10a90e..f9b7a478e0 100644 --- a/selfdrive/car/toyota/carcontroller.py +++ b/selfdrive/car/toyota/carcontroller.py @@ -168,7 +168,7 @@ class CarController(CarControllerBase): if self.frame % 20 == 0 and self.CP.flags & ToyotaFlags.DISABLE_RADAR.value: can_sends.append([0x750, 0, b"\x0F\x02\x3E\x00\x00\x00\x00\x00", 0]) - new_actuators = actuators.copy() + new_actuators = actuators.as_builder() new_actuators.steer = apply_steer / self.params.STEER_MAX new_actuators.steerOutputCan = apply_steer new_actuators.steeringAngleDeg = self.last_angle diff --git a/selfdrive/car/volkswagen/carcontroller.py b/selfdrive/car/volkswagen/carcontroller.py index 5fc47c51c6..a4e0c8946a 100644 --- a/selfdrive/car/volkswagen/carcontroller.py +++ b/selfdrive/car/volkswagen/carcontroller.py @@ -112,7 +112,7 @@ class CarController(CarControllerBase): can_sends.append(self.CCS.create_acc_buttons_control(self.packer_pt, self.ext_bus, CS.gra_stock_values, cancel=CC.cruiseControl.cancel, resume=CC.cruiseControl.resume)) - new_actuators = actuators.copy() + new_actuators = actuators.as_builder() new_actuators.steer = self.apply_steer_last / self.CCP.STEER_MAX new_actuators.steerOutputCan = self.apply_steer_last diff --git a/selfdrive/controls/controlsd.py b/selfdrive/controls/controlsd.py index e3c456766e..d007cd9478 100755 --- a/selfdrive/controls/controlsd.py +++ b/selfdrive/controls/controlsd.py @@ -18,8 +18,7 @@ from openpilot.common.params import Params from openpilot.common.realtime import config_realtime_process, Priority, Ratekeeper, DT_CTRL from openpilot.common.swaglog import cloudlog -from openpilot.selfdrive.car.car_helpers import get_startup_event -from openpilot.selfdrive.car.card import CarD +from openpilot.selfdrive.car.car_helpers import get_car_interface, get_startup_event from openpilot.selfdrive.controls.lib.alertmanager import AlertManager, set_offroad_alert from openpilot.selfdrive.controls.lib.drive_helpers import VCruiseHelper, clip_curvature from openpilot.selfdrive.controls.lib.events import Events, ET @@ -61,16 +60,19 @@ ENABLED_STATES = (State.preEnabled, *ACTIVE_STATES) class Controls: def __init__(self, CI=None): - self.card = CarD(CI) - self.params = Params() - with car.CarParams.from_bytes(self.params.get("CarParams", block=True)) as msg: - # TODO: this shouldn't need to be a builder - self.CP = msg.as_builder() - - self.CI = self.card.CI + if CI is None: + cloudlog.info("controlsd is waiting for CarParams") + with car.CarParams.from_bytes(self.params.get("CarParams", block=True)) as msg: + # TODO: this shouldn't need to be a builder + self.CP = msg.as_builder() + cloudlog.info("controlsd got CarParams") + # Uses car interface helper functions, altering state won't be considered by card for actuation + self.CI = get_car_interface(self.CP) + else: + self.CI, self.CP = CI, CI.CP # Ensure the current branch is cached, otherwise the first iteration of controlsd lags self.branch = get_short_branch() @@ -83,6 +85,9 @@ class Controls: self.log_sock = messaging.sub_sock('androidLog') + # TODO: de-couple controlsd with card/conflate on carState without introducing controls mismatches + self.car_state_sock = messaging.sub_sock('carState', timeout=20) + ignore = self.sensor_packets + ['testJoystick'] if SIMULATION: ignore += ['driverCameraState', 'managerState'] @@ -110,6 +115,7 @@ class Controls: if not self.CP.openpilotLongitudinalControl: self.params.remove("ExperimentalMode") + self.CS_prev = car.CarState.new_message() self.AM = AlertManager() self.events = Events() @@ -161,7 +167,7 @@ class Controls: elif self.CP.passive: self.events.add(EventName.dashcamMode, static=True) - # controlsd is driven by can recv, expected at 100Hz + # controlsd is driven by carState, expected at 100Hz self.rk = Ratekeeper(100, print_delay_threshold=None) def set_initial_state(self): @@ -308,7 +314,7 @@ class Controls: # generic catch-all. ideally, a more specific event should be added above instead has_disable_events = self.events.contains(ET.NO_ENTRY) and (self.events.contains(ET.SOFT_DISABLE) or self.events.contains(ET.IMMEDIATE_DISABLE)) no_system_errors = (not has_disable_events) or (len(self.events) == num_events) - if (not self.sm.all_checks() or self.card.can_rcv_timeout) and no_system_errors: + if (not self.sm.all_checks() or CS.canRcvTimeout) and no_system_errors: if not self.sm.all_alive(): self.events.add(EventName.commIssue) elif not self.sm.all_freq_ok(): @@ -320,7 +326,7 @@ class Controls: 'invalid': [s for s, valid in self.sm.valid.items() if not valid], 'not_alive': [s for s, alive in self.sm.alive.items() if not alive], 'not_freq_ok': [s for s, freq_ok in self.sm.freq_ok.items() if not freq_ok], - 'can_rcv_timeout': self.card.can_rcv_timeout, + 'can_rcv_timeout': CS.canRcvTimeout, } if logs != self.logged_comm_issue: cloudlog.event("commIssue", error=True, **logs) @@ -380,9 +386,10 @@ class Controls: self.events.add(EventName.modeldLagging) def data_sample(self): - """Receive data from sockets and update carState""" + """Receive data from sockets""" - CS = self.card.state_update() + car_state = messaging.recv_one(self.car_state_sock) + CS = car_state.carState if car_state else self.CS_prev self.sm.update(0) @@ -396,9 +403,6 @@ class Controls: if VisionStreamType.VISION_STREAM_WIDE_ROAD not in available_streams: self.sm.ignore_alive.append('wideRoadCameraState') - if not self.CP.passive: - self.card.initialize() - self.initialized = True self.set_initial_state() self.params.put_bool_nonblocking("ControlsReady", True) @@ -709,7 +713,6 @@ class Controls: hudControl.visualAlert = current_alert.visual_alert if not self.CP.passive and self.initialized: - self.card.controls_update(CS, CC) CO = self.sm['carOutput'] if self.CP.steerControlType == car.CarParams.SteerControlType.angle: self.steer_limited = abs(CC.actuators.steeringAngleDeg - CO.actuatorsOutput.steeringAngleDeg) > \ @@ -757,7 +760,6 @@ class Controls: controlsState.cumLagMs = -self.rk.remaining * 1000. controlsState.startMonoTime = int(start_time * 1e9) controlsState.forceDecel = bool(force_decel) - controlsState.canErrorCounter = self.card.can_rcv_cum_timeout_counter controlsState.experimentalMode = self.experimental_mode controlsState.personality = self.personality @@ -807,6 +809,8 @@ class Controls: # Publish data self.publish_logs(CS, start_time, CC, lac_log) + self.CS_prev = CS + def read_personality_param(self): try: return int(self.params.get('LongitudinalPersonality')) diff --git a/selfdrive/controls/tests/test_startup.py b/selfdrive/controls/tests/test_startup.py index 23cc96a2e4..5b69a7cd82 100644 --- a/selfdrive/controls/tests/test_startup.py +++ b/selfdrive/controls/tests/test_startup.py @@ -87,6 +87,7 @@ def test_startup_alert(expected_event, car_model, fw_versions, brand): os.environ['SKIP_FW_QUERY'] = '1' managed_processes['controlsd'].start() + managed_processes['card'].start() assert pm.wait_for_readers_to_update('can', 5) pm.send('can', can_list_to_can_capnp([[0, 0, b"", 0]])) @@ -104,7 +105,7 @@ def test_startup_alert(expected_event, car_model, fw_versions, brand): msgs = [[addr, 0, b'\x00'*length, 0] for addr, length in finger.items()] for _ in range(1000): - # controlsd waits for boardd to echo back that it has changed the multiplexing mode + # card waits for boardd to echo back that it has changed the multiplexing mode if not params.get_bool("ObdMultiplexingChanged"): params.put_bool("ObdMultiplexingChanged", True) diff --git a/selfdrive/manager/process_config.py b/selfdrive/manager/process_config.py index c2aa3b44e2..a8ab204da4 100644 --- a/selfdrive/manager/process_config.py +++ b/selfdrive/manager/process_config.py @@ -64,6 +64,7 @@ procs = [ PythonProcess("calibrationd", "selfdrive.locationd.calibrationd", only_onroad), PythonProcess("torqued", "selfdrive.locationd.torqued", only_onroad), PythonProcess("controlsd", "selfdrive.controls.controlsd", only_onroad), + PythonProcess("card", "selfdrive.car.card", only_onroad), PythonProcess("deleter", "system.loggerd.deleter", always_run), PythonProcess("dmonitoringd", "selfdrive.monitoring.dmonitoringd", driverview, enabled=(not PC or WEBCAM)), PythonProcess("qcomgpsd", "system.qcomgpsd.qcomgpsd", qcomgps, enabled=TICI), diff --git a/selfdrive/test/process_replay/migration.py b/selfdrive/test/process_replay/migration.py index be78c92d73..c8d0504def 100644 --- a/selfdrive/test/process_replay/migration.py +++ b/selfdrive/test/process_replay/migration.py @@ -14,6 +14,7 @@ def migrate_all(lr, old_logtime=False, manager_states=False, panda_states=False, msgs = migrate_carParams(msgs, old_logtime) msgs = migrate_gpsLocation(msgs) msgs = migrate_deviceState(msgs) + msgs = migrate_carOutput(msgs) if manager_states: msgs = migrate_managerState(msgs) if panda_states: @@ -69,6 +70,23 @@ def migrate_deviceState(lr): return all_msgs +def migrate_carOutput(lr): + # migration needed only for routes before carOutput + if any(msg.which() == 'carOutput' for msg in lr): + return lr + + all_msgs = [] + for msg in lr: + if msg.which() == 'carControl': + co = messaging.new_message('carOutput') + co.valid = msg.valid + co.logMonoTime = msg.logMonoTime + co.carOutput.actuatorsOutput = msg.carControl.actuatorsOutputDEPRECATED + all_msgs.append(co.as_reader()) + all_msgs.append(msg) + return all_msgs + + def migrate_pandaStates(lr): all_msgs = [] # TODO: safety param migration should be handled automatically diff --git a/selfdrive/test/process_replay/process_replay.py b/selfdrive/test/process_replay/process_replay.py index f55f2c4e2a..fd1d753467 100755 --- a/selfdrive/test/process_replay/process_replay.py +++ b/selfdrive/test/process_replay/process_replay.py @@ -317,12 +317,12 @@ class ProcessContainer: return output_msgs -def controlsd_fingerprint_callback(rc, pm, msgs, fingerprint): +def card_fingerprint_callback(rc, pm, msgs, fingerprint): print("start fingerprinting") params = Params() canmsgs = [msg for msg in msgs if msg.which() == "can"][:300] - # controlsd expects one arbitrary can and pandaState + # card expects one arbitrary can and pandaState rc.send_sync(pm, "can", messaging.new_message("can", 1)) pm.send("pandaStates", messaging.new_message("pandaStates", 1)) rc.send_sync(pm, "can", messaging.new_message("can", 1)) @@ -356,12 +356,20 @@ def get_car_params_callback(rc, pm, msgs, fingerprint): for m in canmsgs[:300]: can.send(m.as_builder().to_bytes()) _, CP = get_car(can, sendcan, Params().get_bool("ExperimentalLongitudinalEnabled")) + + if not params.get_bool("DisengageOnAccelerator"): + CP.alternativeExperience |= ALTERNATIVE_EXPERIENCE.DISABLE_DISENGAGE_ON_GAS + params.put("CarParams", CP.to_bytes()) return CP def controlsd_rcv_callback(msg, cfg, frame): - # no sendcan until controlsd is initialized + return (frame - 1) == 0 or msg.which() == 'carState' + + +def card_rcv_callback(msg, cfg, frame): + # no sendcan until card is initialized if msg.which() != "can": return False @@ -461,18 +469,28 @@ CONFIGS = [ ProcessConfig( proc_name="controlsd", pubs=[ - "can", "deviceState", "pandaStates", "peripheralState", "liveCalibration", "driverMonitoringState", + "carState", "deviceState", "pandaStates", "peripheralState", "liveCalibration", "driverMonitoringState", "longitudinalPlan", "liveLocationKalman", "liveParameters", "radarState", "modelV2", "driverCameraState", "roadCameraState", "wideRoadCameraState", "managerState", - "testJoystick", "liveTorqueParameters", "accelerometer", "gyroscope" + "testJoystick", "liveTorqueParameters", "accelerometer", "gyroscope", "carOutput" ], - subs=["controlsState", "carState", "carControl", "carOutput", "sendcan", "onroadEvents", "carParams"], + subs=["controlsState", "carControl", "onroadEvents"], ignore=["logMonoTime", "controlsState.startMonoTime", "controlsState.cumLagMs"], config_callback=controlsd_config_callback, - init_callback=controlsd_fingerprint_callback, + init_callback=get_car_params_callback, should_recv_callback=controlsd_rcv_callback, tolerance=NUMPY_TOLERANCE, processing_time=0.004, + ), + ProcessConfig( + proc_name="card", + pubs=["pandaStates", "carControl", "onroadEvents", "can"], + subs=["sendcan", "carState", "carParams", "carOutput"], + ignore=["logMonoTime", "carState.cumLagMs"], + init_callback=card_fingerprint_callback, + should_recv_callback=card_rcv_callback, + tolerance=NUMPY_TOLERANCE, + processing_time=0.004, main_pub="can", ), ProcessConfig( diff --git a/selfdrive/test/process_replay/ref_commit b/selfdrive/test/process_replay/ref_commit index 2af30b5b47..dc7e59b54f 100644 --- a/selfdrive/test/process_replay/ref_commit +++ b/selfdrive/test/process_replay/ref_commit @@ -1 +1 @@ -cc4e23ca0fceb300a3048c187ae9bc793794c095 \ No newline at end of file +c84b55c256ccb0ee042643a8a7f7f4dc429ff157 \ No newline at end of file diff --git a/selfdrive/test/process_replay/test_fuzzy.py b/selfdrive/test/process_replay/test_fuzzy.py index d295092b20..6bcc94911f 100755 --- a/selfdrive/test/process_replay/test_fuzzy.py +++ b/selfdrive/test/process_replay/test_fuzzy.py @@ -12,7 +12,7 @@ import openpilot.selfdrive.test.process_replay.process_replay as pr # These processes currently fail because of unrealistic data breaking assumptions # that openpilot makes causing error with NaN, inf, int size, array indexing ... # TODO: Make each one testable -NOT_TESTED = ['controlsd', 'plannerd', 'calibrationd', 'dmonitoringd', 'paramsd', 'dmonitoringmodeld', 'modeld'] +NOT_TESTED = ['controlsd', 'card', 'plannerd', 'calibrationd', 'dmonitoringd', 'paramsd', 'dmonitoringmodeld', 'modeld'] TEST_CASES = [(cfg.proc_name, copy.deepcopy(cfg)) for cfg in pr.CONFIGS if cfg.proc_name not in NOT_TESTED] diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index b3dcbb03db..c0a425b599 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -41,23 +41,23 @@ source_segments = [ ] segments = [ - ("BODY", "regen02ECB79BACC|2024-05-18--04-29-29--0"), - ("HYUNDAI", "regen845DC1916E6|2024-05-18--04-30-52--0"), - ("HYUNDAI2", "regenBAE0915FE22|2024-05-18--04-33-11--0"), - ("TOYOTA", "regen1108D19FC2E|2024-05-18--04-34-34--0"), - ("TOYOTA2", "regen846521F39C7|2024-05-18--04-35-58--0"), - ("TOYOTA3", "regen788C3623D11|2024-05-18--04-38-21--0"), - ("HONDA", "regenDE43F170E99|2024-05-18--04-39-47--0"), - ("HONDA2", "regen1EE0FA383C3|2024-05-18--04-41-12--0"), - ("CHRYSLER", "regen9C5A30F471C|2024-05-18--04-42-36--0"), - ("RAM", "regenCCA313D117D|2024-05-18--04-44-53--0"), - ("SUBARU", "regenA41511F882A|2024-05-18--04-47-14--0"), - ("GM", "regen9D7B9CE4A66|2024-05-18--04-48-36--0"), - ("GM2", "regen07CECA52D41|2024-05-18--04-50-55--0"), - ("NISSAN", "regen2D6B856D0AE|2024-05-18--04-52-17--0"), - ("VOLKSWAGEN", "regen2D3AC6A6F05|2024-05-18--04-53-41--0"), - ("MAZDA", "regen0D5A777DD16|2024-05-18--04-56-02--0"), - ("FORD", "regen235D0937965|2024-05-18--04-58-16--0"), + ("BODY", "regen29FD9FF7760|2024-05-21--06-58-51--0"), + ("HYUNDAI", "regen0B1B76A1C27|2024-05-21--06-57-53--0"), + ("HYUNDAI2", "regen3BB55FA5E20|2024-05-21--06-59-03--0"), + ("TOYOTA", "regenF6FB954C1E2|2024-05-21--06-57-53--0"), + ("TOYOTA2", "regen0AC637CE7BA|2024-05-21--06-57-54--0"), + ("TOYOTA3", "regenC7BE3FAE496|2024-05-21--06-59-01--0"), + ("HONDA", "regen58E9F8B695A|2024-05-21--06-57-55--0"), + ("HONDA2", "regen8695608EB15|2024-05-21--06-57-55--0"), + ("CHRYSLER", "regenB0F8C25C902|2024-05-21--06-59-47--0"), + ("RAM", "regenB3B2C7A105B|2024-05-21--07-00-47--0"), + ("SUBARU", "regen860FD736DCC|2024-05-21--07-00-50--0"), + ("GM", "regen8CB3048DEB9|2024-05-21--06-59-49--0"), + ("GM2", "regen379D446541D|2024-05-21--07-00-51--0"), + ("NISSAN", "regen24871108F80|2024-05-21--07-00-38--0"), + ("VOLKSWAGEN", "regenF390392F275|2024-05-21--07-00-52--0"), + ("MAZDA", "regenE5A36020581|2024-05-21--07-01-51--0"), + ("FORD", "regenDC288ED0D78|2024-05-21--07-02-18--0"), ] # dashcamOnly makes don't need to be tested until a full port is done @@ -109,7 +109,7 @@ def test_process(cfg, lr, segment, ref_log_path, new_log_path, ignore_fields=Non if not check_openpilot_enabled(log_msgs): return f"Route did not enable at all or for long enough: {new_log_path}", log_msgs - if cfg.proc_name != 'ubloxd' or segment != 'regenBAE0915FE22|2024-05-18--04-33-11--0': + if cfg.proc_name != 'ubloxd' or segment != 'regen3BB55FA5E20|2024-05-21--06-59-03--0': seen_msgs = {m.which() for m in log_msgs} expected_msgs = set(cfg.subs) if seen_msgs != expected_msgs: diff --git a/selfdrive/test/test_onroad.py b/selfdrive/test/test_onroad.py index 14f7773302..038bba7b2e 100755 --- a/selfdrive/test/test_onroad.py +++ b/selfdrive/test/test_onroad.py @@ -35,7 +35,8 @@ CPU usage budget MAX_TOTAL_CPU = 250. # total for all 8 cores PROCS = { # Baseline CPU usage by process - "selfdrive.controls.controlsd": 46.0, + "selfdrive.controls.controlsd": 32.0, + "selfdrive.car.card": 22.0, "./loggerd": 14.0, "./encoderd": 17.0, "./camerad": 14.5, From e836845f02adc7abab05de8852859c315780c9d3 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 21 May 2024 01:19:55 -0700 Subject: [PATCH 22/26] update TOTAL_SCONS_NODES --- selfdrive/manager/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfdrive/manager/build.py b/selfdrive/manager/build.py index 859d0920c3..ac7822d51b 100755 --- a/selfdrive/manager/build.py +++ b/selfdrive/manager/build.py @@ -14,7 +14,7 @@ from openpilot.system.version import get_build_metadata MAX_CACHE_SIZE = 4e9 if "CI" in os.environ else 2e9 CACHE_DIR = Path("/data/scons_cache" if AGNOS else "/tmp/scons_cache") -TOTAL_SCONS_NODES = 2560 +TOTAL_SCONS_NODES = 2410 MAX_BUILD_PROGRESS = 100 def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None: From b9c1c1dd37a195660ce44fbb24aaf5057155a6bf Mon Sep 17 00:00:00 2001 From: Shotaro Watanabe Date: Wed, 22 May 2024 02:59:15 +0900 Subject: [PATCH 23/26] devcontainer: mount /dev (#32500) --- .devcontainer/devcontainer.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 7174ee2800..3f5b38d12e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -14,6 +14,7 @@ "force_color_prompt": "1" }, "runArgs": [ + "--volume=/dev:/dev", "--volume=/tmp/.X11-unix:/tmp/.X11-unix", "--volume=${localWorkspaceFolder}/.devcontainer/.host/.Xauthority:/home/batman/.Xauthority", "--volume=${localEnv:HOME}/.comma:/home/batman/.comma", @@ -49,4 +50,4 @@ "mounts": [ "type=volume,source=scons_cache,target=/tmp/scons_cache" ] -} \ No newline at end of file +} From e29ed6849e4bd8e53b6ce03cfb860af071d75b16 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Tue, 21 May 2024 16:37:24 -0500 Subject: [PATCH 24/26] [bot] Fingerprints: add missing FW versions from new users (#32501) Export fingerprints --- selfdrive/car/hyundai/fingerprints.py | 1 + selfdrive/car/toyota/fingerprints.py | 1 + 2 files changed, 2 insertions(+) diff --git a/selfdrive/car/hyundai/fingerprints.py b/selfdrive/car/hyundai/fingerprints.py index e0680ba829..8e574f7a25 100644 --- a/selfdrive/car/hyundai/fingerprints.py +++ b/selfdrive/car/hyundai/fingerprints.py @@ -864,6 +864,7 @@ FW_VERSIONS = { b'\xf1\x00CN7_ SCC FNCUP 1.00 1.01 99110-AA000 ', ], (Ecu.eps, 0x7d4, None): [ + b'\xf1\x00CN7 MDPS C 1.00 1.06 56310/AA050 4CNDC106', b'\xf1\x00CN7 MDPS C 1.00 1.06 56310/AA070 4CNDC106', b'\xf1\x00CN7 MDPS C 1.00 1.06 56310AA050\x00 4CNDC106', b'\xf1\x00CN7 MDPS C 1.00 1.07 56310AA050\x00 4CNDC107', diff --git a/selfdrive/car/toyota/fingerprints.py b/selfdrive/car/toyota/fingerprints.py index d239294392..38105e0d9d 100644 --- a/selfdrive/car/toyota/fingerprints.py +++ b/selfdrive/car/toyota/fingerprints.py @@ -1476,6 +1476,7 @@ FW_VERSIONS = { b'\x01896630E41200\x00\x00\x00\x00', b'\x01896630E41500\x00\x00\x00\x00', b'\x01896630EA3100\x00\x00\x00\x00', + b'\x01896630EA3300\x00\x00\x00\x00', b'\x01896630EA3400\x00\x00\x00\x00', b'\x01896630EA4100\x00\x00\x00\x00', b'\x01896630EA4300\x00\x00\x00\x00', From 527cd74b21a182d423f41db2a04e5f4af3632d16 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 21 May 2024 15:25:50 -0700 Subject: [PATCH 25/26] CI: cleanup PR review jobs (#32503) --- .github/workflows/auto_pr_review.yaml | 222 +++----------------------- 1 file changed, 24 insertions(+), 198 deletions(-) diff --git a/.github/workflows/auto_pr_review.yaml b/.github/workflows/auto_pr_review.yaml index 8316fedda0..447e559c90 100644 --- a/.github/workflows/auto_pr_review.yaml +++ b/.github/workflows/auto_pr_review.yaml @@ -5,7 +5,7 @@ on: jobs: labeler: - name: apply labels + name: review permissions: contents: read pull-requests: write @@ -14,17 +14,17 @@ jobs: - uses: actions/checkout@v4 with: submodules: false + + # Label PRs - uses: actions/labeler@v5.0.0 with: dot: true configuration-path: .github/labeler.yaml - pr_branch_check: - name: check branch - runs-on: ubuntu-latest - if: github.repository == 'commaai/openpilot' - steps: - - uses: Vankka/pr-target-branch-action@def32ec9d93514138d6ac0132ee62e120a72aed5 + # Check PR target branch + - name: check branch + uses: Vankka/pr-target-branch-action@def32ec9d93514138d6ac0132ee62e120a72aed5 + if: github.repository == 'commaai/openpilot' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -34,194 +34,20 @@ jobs: already-exists-action: close_this already-exists-comment: "Your PR should be made against the `master` branch" - comment: - runs-on: ubuntu-latest - steps: - - name: comment - uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6 - if: github.event.pull_request.head.repo.full_name != 'commaai/openpilot' - with: - message: | - - Thanks for contributing to openpilot! In order for us to review your PR as quickly as possible, check the following: - * Convert your PR to a draft unless it's ready to review - * Read the [contributing docs](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md) - * Before marking as "ready for review", ensure: - * the goal is clearly stated in the description - * all the tests are passing - * the change is [something we merge](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md#what-gets-merged) - * include a route or your device' dongle ID if relevant - comment_tag: run_id - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - check-pr-template: - runs-on: ubuntu-latest - permissions: - contents: read - issues: write - pull-requests: write - actions: read - if: false && github.event.pull_request.head.repo.full_name != 'commaai/openpilot' - steps: - - uses: actions/github-script@v7 - with: - script: | - // Comment to add to the PR if no template has been used - const NO_TEMPLATE_MESSAGE = - "It looks like you didn't use one of the Pull Request templates. Please check [the contributing docs](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md). \ - Also make sure that you didn't modify any of the checkboxes or headings within the template."; - // body data for future requests - const body_data = { - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - }; - - // Utility function to extract all headings - const extractHeadings = (markdown) => { - const headingRegex = /^(#{1,6})\s+(.+)$/gm; - const boldTextRegex = /^(?:\*\*|__)(.+?)(?:\*\*|__)\s*$/gm; - const headings = []; - let headingMatch; - while ((headingMatch = headingRegex.exec(markdown))) { - headings.push(headingMatch[2].trim()); - } - let boldMatch; - while ((boldMatch = boldTextRegex.exec(markdown))) { - headings.push(boldMatch[1].trim()); - } - return headings; - }; - - // Utility function to extract all check box descriptions - const extractCheckBoxTexts = (markdown) => { - const checkboxRegex = /^\s*-\s*\[( |x)\]\s+(.+)$/gm; - const checkboxes = []; - let match; - while ((match = checkboxRegex.exec(markdown))) { - checkboxes.push(match[2].trim()); - } - return checkboxes; - }; - - // Utility function to check if a list is a subset of another list - isSubset = (subset, superset) => { - return subset.every((item) => superset.includes(item)); - }; - - // Utility function to check if a list of checkboxes is a subset of another list of checkboxes - isCheckboxSubset = (templateCheckBoxTexts, prTextCheckBoxTexts) => { - // Check if each template checkbox text is a substring of at least one PR checkbox text - // (user should be allowed to add additional text) - return templateCheckBoxTexts.every((item) => prTextCheckBoxTexts.some((element) => element.includes(item))) - } - - // Get filenames of all currently checked-in PR templates - const template_contents = await github.rest.repos.getContent({ - owner: context.repo.owner, - repo: context.repo.repo, - path: ".github/PULL_REQUEST_TEMPLATE", - }); - var template_filenames = []; - for (const content of template_contents.data) { - template_filenames.push(content.path); - } - console.debug("Received template filenames: " + template_filenames); - // Retrieve templates - var templates = []; - for (const template_filename of template_filenames) { - const template_response = await github.rest.repos.getContent({ - owner: context.repo.owner, - repo: context.repo.repo, - path: template_filename, - }); - // Convert Base64 content back - const decoded_template = atob(template_response.data.content); - const headings = extractHeadings(decoded_template); - const checkboxes = extractCheckBoxTexts(decoded_template); - if (!headings.length && !checkboxes.length) { - console.warn( - "Invalid template! Contains neither headings nor checkboxes, ignoring it: \n" + - decoded_template - ); - } else { - templates.push({ headings: headings, checkboxes: checkboxes }); - } - } - // Retrieve the PR Body - const pull_request = await github.rest.issues.get({ - ...body_data, - }); - const pull_request_text = pull_request.data.body; - console.debug("Received Pull Request body: \n" + pull_request_text); - - /* Check if the PR Body matches one of the templates - A template is defined by all headings and checkboxes it contains - We extract all Headings and Checkboxes from the PR text and check if any of the templates is a subset of that - */ - const pr_headings = extractHeadings(pull_request_text); - const pr_checkboxes = extractCheckBoxTexts(pull_request_text); - console.debug("Found Headings in PR body:\n" + pr_headings); - console.debug("Found Checkboxes in PR body:\n" + pr_checkboxes); - var template_found = false; - // Iterate over each template to check if it applies - for (const template of templates) { - console.log( - "Checking for headings: [" + - template.headings + - "] and checkboxes: [" + - template.checkboxes + "]" - ); - if ( - isCheckboxSubset(template.checkboxes, pr_checkboxes) && - isSubset(template.headings, pr_headings) - ) { - console.debug("Found matching template!"); - template_found = true; - } - } - - // List comments from previous runs - var existing_comments = []; - const comments = await github.rest.issues.listComments({ - ...body_data, - }); - for (const comment of comments.data) { - if (comment.body === NO_TEMPLATE_MESSAGE) { - existing_comments.push(comment); - } - } - - // Add a comment to the PR that it is not using a the template (but only if this comment does not exist already) - if (!template_found) { - var comment_already_sent = false; - - // Add an 'in-bot-review' label since this PR doesn't have the template - github.rest.issues.addLabels({ - ...body_data, - labels: ["in-bot-review"], - }); - - if (existing_comments.length < 1) { - github.rest.issues.createComment({ - ...body_data, - body: NO_TEMPLATE_MESSAGE, - }); - } - } else { - // If template has been found, delete any old comment about missing template - for (const existing_comment of existing_comments) { - github.rest.issues.deleteComment({ - ...body_data, - comment_id: existing_comment.id, - }); - } - // Remove the 'in-bot-review' label after the review is done and the PR has passed - github.rest.issues.removeLabel({ - ...body_data, - name: "in-bot-review", - }).catch((error) => { - console.log("Label 'in-bot-review' not found, ignoring"); - }); - } - + # Welcome comment + - name: comment + uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6 + if: github.event.pull_request.head.repo.full_name != 'commaai/openpilot' + with: + message: | + + Thanks for contributing to openpilot! In order for us to review your PR as quickly as possible, check the following: + * Convert your PR to a draft unless it's ready to review + * Read the [contributing docs](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md) + * Before marking as "ready for review", ensure: + * the goal is clearly stated in the description + * all the tests are passing + * the change is [something we merge](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md#what-gets-merged) + * include a route or your device' dongle ID if relevant + comment_tag: run_id + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 406f30add493006b99569bfbf49562e3192038cc Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 21 May 2024 15:40:00 -0700 Subject: [PATCH 26/26] more apt dependency cleanup (#32502) rm --- tools/install_ubuntu_dependencies.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/install_ubuntu_dependencies.sh b/tools/install_ubuntu_dependencies.sh index 0e3d453ce4..1f8ef24eaf 100755 --- a/tools/install_ubuntu_dependencies.sh +++ b/tools/install_ubuntu_dependencies.sh @@ -41,8 +41,6 @@ function install_ubuntu_common_requirements() { libgles2-mesa-dev \ libglfw3-dev \ libglib2.0-0 \ - libomp-dev \ - libpng16-16 \ libqt5charts5-dev \ libncurses5-dev \ libssl-dev \