From af774d894e6698f832c1d813587e3d7dabad0ea7 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Thu, 3 Oct 2024 15:20:58 -0700 Subject: [PATCH] Deprecate car onroadEvents (#33687) * new OnroadEvent struct * and migrate python * more forgotten * re-index new OnroadEvent struct * fix! * more missing * migrate onroadEvents migrate onroadEvents * migrate dm events * hacks to get proc replay to say succeeded * Revert "hacks to get proc replay to say succeeded" This reverts commit 0bb8803e5755d606ae9f09da5395d9f50678c7c7. * update refs --- cereal/car.capnp | 9 +- cereal/log.capnp | 120 ++++++++++++++++++++- selfdrive/car/car_specific.py | 4 +- selfdrive/car/card.py | 4 +- selfdrive/car/tests/test_models.py | 2 +- selfdrive/debug/cycle_alerts.py | 2 +- selfdrive/monitoring/helpers.py | 4 +- selfdrive/monitoring/test_monitoring.py | 4 +- selfdrive/selfdrived/events.py | 4 +- selfdrive/selfdrived/selfdrived.py | 2 +- selfdrive/selfdrived/tests/test_alerts.py | 2 +- selfdrive/test/process_replay/migration.py | 32 +++++- selfdrive/test/process_replay/ref_commit | 2 +- selfdrive/test/test_onroad.py | 4 +- selfdrive/test/test_time_to_onroad.py | 4 +- 15 files changed, 171 insertions(+), 28 deletions(-) diff --git a/cereal/car.capnp b/cereal/car.capnp index 326f1eeef8..c8474f431b 100644 --- a/cereal/car.capnp +++ b/cereal/car.capnp @@ -5,9 +5,8 @@ $Cxx.namespace("cereal"); # ******* events causing controls state machine transition ******* -# FIXME: OnroadEvent shouldn't be in car.capnp, but can't immediately -# move due to being referenced by structs in this file -struct OnroadEvent @0x9b1657f34caf3ad3 { +# IMPORTANT: This struct is to not be modified so old logs can be parsed +struct OnroadEventDEPRECATED @0x9b1657f34caf3ad3 { name @0 :EventName; # event types @@ -289,12 +288,12 @@ struct CarState { } # deprecated - errorsDEPRECATED @0 :List(OnroadEvent.EventName); + errorsDEPRECATED @0 :List(OnroadEventDEPRECATED.EventName); brakeLightsDEPRECATED @19 :Bool; steeringRateLimitedDEPRECATED @29 :Bool; canMonoTimesDEPRECATED @12: List(UInt64); canRcvTimeoutDEPRECATED @49 :Bool; - eventsDEPRECATED @13 :List(OnroadEvent); + eventsDEPRECATED @13 :List(OnroadEventDEPRECATED); } # ******* radar state @ 20hz ******* diff --git a/cereal/log.capnp b/cereal/log.capnp index 3d5f7d9d7f..c35219b3ac 100644 --- a/cereal/log.capnp +++ b/cereal/log.capnp @@ -17,6 +17,118 @@ struct Map(Key, Value) { } } +struct OnroadEvent @0xc4fa6047f024e718 { + name @0 :EventName; + + # event types + enable @1 :Bool; + noEntry @2 :Bool; + warning @3 :Bool; # alerts presented only when enabled or soft disabling + userDisable @4 :Bool; + softDisable @5 :Bool; + immediateDisable @6 :Bool; + preEnable @7 :Bool; + permanent @8 :Bool; # alerts presented regardless of openpilot state + overrideLateral @10 :Bool; + overrideLongitudinal @9 :Bool; + + enum EventName @0x91f1992a1f77fb03 { + canError @0; + steerUnavailable @1; + wrongGear @2; + doorOpen @3; + seatbeltNotLatched @4; + espDisabled @5; + wrongCarMode @6; + steerTempUnavailable @7; + reverseGear @8; + buttonCancel @9; + buttonEnable @10; + pedalPressed @11; # exits active state + preEnableStandstill @12; # added during pre-enable state with brake + gasPressedOverride @13; # added when user is pressing gas with no disengage on gas + steerOverride @14; + cruiseDisabled @15; + speedTooLow @16; + outOfSpace @17; + overheat @18; + calibrationIncomplete @19; + calibrationInvalid @20; + calibrationRecalibrating @21; + controlsMismatch @22; + pcmEnable @23; + pcmDisable @24; + radarFault @25; + brakeHold @26; + parkBrake @27; + manualRestart @28; + joystickDebug @29; + longitudinalManeuver @30; + steerTempUnavailableSilent @31; + resumeRequired @32; + preDriverDistracted @33; + promptDriverDistracted @34; + driverDistracted @35; + preDriverUnresponsive @36; + promptDriverUnresponsive @37; + driverUnresponsive @38; + belowSteerSpeed @39; + lowBattery @40; + accFaulted @41; + sensorDataInvalid @42; + commIssue @43; + commIssueAvgFreq @44; + tooDistracted @45; + posenetInvalid @46; + soundsUnavailable @47; + preLaneChangeLeft @48; + preLaneChangeRight @49; + laneChange @50; + lowMemory @51; + stockAeb @52; + ldw @53; + carUnrecognized @54; + invalidLkasSetting @55; + speedTooHigh @56; + laneChangeBlocked @57; + relayMalfunction @58; + stockFcw @59; + startup @60; + startupNoCar @61; + startupNoControl @62; + startupNoSecOcKey @63; + startupMaster @64; + fcw @65; + steerSaturated @66; + belowEngageSpeed @67; + noGps @68; + wrongCruiseMode @69; + modeldLagging @70; + deviceFalling @71; + fanMalfunction @72; + cameraMalfunction @73; + cameraFrameRate @74; + processNotRunning @75; + dashcamMode @76; + selfdriveInitializing @77; + usbError @78; + cruiseMismatch @79; + canBusMissing @80; + selfdrivedLagging @81; + resumeBlocked @82; + steerTimeLimit @83; + vehicleSensorsInvalid @84; + locationdTemporaryError @85; + locationdPermanentError @86; + paramsdTemporaryError @87; + paramsdPermanentError @88; + actuatorsApiUnavailable @89; + espActive @90; + personalityChanged @91; + aeb @92; + } +} + enum LongitudinalPersonality { aggressive @0; standard @1; @@ -1157,7 +1269,7 @@ struct LongitudinalPlan @0xe00b5b3eba12876c { radarValidDEPRECATED @28 :Bool; radarCanErrorDEPRECATED @30 :Bool; commIssueDEPRECATED @31 :Bool; - eventsDEPRECATED @13 :List(Car.OnroadEvent); + eventsDEPRECATED @13 :List(Car.OnroadEventDEPRECATED); gpsTrajectoryDEPRECATED @12 :GpsTrajectory; gpsPlannerActiveDEPRECATED @19 :Bool; personalityDEPRECATED @36 :LongitudinalPersonality; @@ -2072,7 +2184,7 @@ struct DriverStateDEPRECATED @0xb83c6cc593ed0a00 { } struct DriverMonitoringState @0xb83cda094a1da284 { - events @0 :List(Car.OnroadEvent); + events @18 :List(OnroadEvent); faceDetected @1 :Bool; isDistracted @2 :Bool; distractedType @17 :UInt32; @@ -2091,6 +2203,7 @@ struct DriverMonitoringState @0xb83cda094a1da284 { isPreviewDEPRECATED @15 :Bool; rhdCheckedDEPRECATED @5 :Bool; + eventsDEPRECATED @0 :List(Car.OnroadEventDEPRECATED); } struct Boot { @@ -2369,7 +2482,7 @@ struct Event { liveTorqueParameters @94 :LiveTorqueParametersData; cameraOdometry @63 :CameraOdometry; thumbnail @66: Thumbnail; - onroadEvents @68: List(Car.OnroadEvent); + onroadEvents @134: List(OnroadEvent); carParams @69: Car.CarParams; driverMonitoringState @71: DriverMonitoringState; livePose @129 :LivePose; @@ -2484,5 +2597,6 @@ struct Event { uiPlanDEPRECATED @106 :UiPlan; liveLocationKalmanDEPRECATED @72 :LiveLocationKalman; liveTracksDEPRECATED @16 :List(LiveTracksDEPRECATED); + onroadEventsDEPRECATED @68: List(Car.OnroadEventDEPRECATED); } } diff --git a/selfdrive/car/car_specific.py b/selfdrive/car/car_specific.py index 23719a8c4f..94afec50e2 100644 --- a/selfdrive/car/car_specific.py +++ b/selfdrive/car/car_specific.py @@ -1,5 +1,5 @@ from collections import deque -from cereal import car +from cereal import car, log import cereal.messaging as messaging from opendbc.car import DT_CTRL, structs from opendbc.car.interfaces import MAX_CTRL_SPEED @@ -11,7 +11,7 @@ from openpilot.selfdrive.selfdrived.events import Events ButtonType = structs.CarState.ButtonEvent.Type GearShifter = structs.CarState.GearShifter -EventName = car.OnroadEvent.EventName +EventName = log.OnroadEvent.EventName NetworkLocation = structs.CarParams.NetworkLocation diff --git a/selfdrive/car/card.py b/selfdrive/car/card.py index 380d5df431..0afd0e7323 100755 --- a/selfdrive/car/card.py +++ b/selfdrive/car/card.py @@ -5,7 +5,7 @@ import threading import cereal.messaging as messaging -from cereal import car +from cereal import car, log from panda import ALTERNATIVE_EXPERIENCE @@ -25,7 +25,7 @@ from openpilot.selfdrive.car.helpers import convert_carControl, convert_to_capnp REPLAY = "REPLAY" in os.environ -EventName = car.OnroadEvent.EventName +EventName = log.OnroadEvent.EventName # forward carlog.addHandler(ForwardingHandler(cloudlog)) diff --git a/selfdrive/car/tests/test_models.py b/selfdrive/car/tests/test_models.py index 10620c146f..ef54631e54 100644 --- a/selfdrive/car/tests/test_models.py +++ b/selfdrive/car/tests/test_models.py @@ -31,7 +31,7 @@ from openpilot.tools.lib.route import SegmentName from panda.tests.libpanda import libpanda_py -EventName = car.OnroadEvent.EventName +EventName = log.OnroadEvent.EventName PandaType = log.PandaState.PandaType SafetyModel = car.CarParams.SafetyModel diff --git a/selfdrive/debug/cycle_alerts.py b/selfdrive/debug/cycle_alerts.py index ec32306436..11e75a7a8e 100755 --- a/selfdrive/debug/cycle_alerts.py +++ b/selfdrive/debug/cycle_alerts.py @@ -10,7 +10,7 @@ from openpilot.selfdrive.selfdrived.events import ET, Events from openpilot.selfdrive.selfdrived.alertmanager import AlertManager from openpilot.system.manager.process_config import managed_processes -EventName = car.OnroadEvent.EventName +EventName = log.OnroadEvent.EventName def randperc() -> float: return 100. * random.random() diff --git a/selfdrive/monitoring/helpers.py b/selfdrive/monitoring/helpers.py index 72df724532..88665778a2 100644 --- a/selfdrive/monitoring/helpers.py +++ b/selfdrive/monitoring/helpers.py @@ -1,6 +1,6 @@ from math import atan2 -from cereal import car +from cereal import car, log import cereal.messaging as messaging from openpilot.selfdrive.selfdrived.events import Events from openpilot.common.numpy_fast import interp @@ -9,7 +9,7 @@ from openpilot.common.filter_simple import FirstOrderFilter from openpilot.common.stat_live import RunningStatFilter from openpilot.common.transformations.camera import DEVICE_CAMERAS -EventName = car.OnroadEvent.EventName +EventName = log.OnroadEvent.EventName # ****************************************************************************************** # NOTE: To fork maintainers. diff --git a/selfdrive/monitoring/test_monitoring.py b/selfdrive/monitoring/test_monitoring.py index 31ae98c638..2a20b20dc1 100644 --- a/selfdrive/monitoring/test_monitoring.py +++ b/selfdrive/monitoring/test_monitoring.py @@ -1,10 +1,10 @@ import numpy as np -from cereal import car, log +from cereal import log from openpilot.common.realtime import DT_DMON from openpilot.selfdrive.monitoring.helpers import DriverMonitoring, DRIVER_MONITOR_SETTINGS -EventName = car.OnroadEvent.EventName +EventName = log.OnroadEvent.EventName dm_settings = DRIVER_MONITOR_SETTINGS() TEST_TIMESPAN = 120 # seconds diff --git a/selfdrive/selfdrived/events.py b/selfdrive/selfdrived/events.py index 070ca60ca2..7be514e350 100755 --- a/selfdrive/selfdrived/events.py +++ b/selfdrive/selfdrived/events.py @@ -16,7 +16,7 @@ AlertSize = log.SelfdriveState.AlertSize AlertStatus = log.SelfdriveState.AlertStatus VisualAlert = car.CarControl.HUDControl.VisualAlert AudibleAlert = car.CarControl.HUDControl.AudibleAlert -EventName = car.OnroadEvent.EventName +EventName = log.OnroadEvent.EventName # Alert priorities @@ -98,7 +98,7 @@ class Events: def to_msg(self): ret = [] for event_name in self.events: - event = car.OnroadEvent.new_message() + event = log.OnroadEvent.new_message() event.name = event_name for event_type in EVENTS.get(event_name, {}): setattr(event, event_type, True) diff --git a/selfdrive/selfdrived/selfdrived.py b/selfdrive/selfdrived/selfdrived.py index 160117faa8..8b7575d35d 100755 --- a/selfdrive/selfdrived/selfdrived.py +++ b/selfdrive/selfdrived/selfdrived.py @@ -35,7 +35,7 @@ State = log.SelfdriveState.OpenpilotState PandaType = log.PandaState.PandaType LaneChangeState = log.LaneChangeState LaneChangeDirection = log.LaneChangeDirection -EventName = car.OnroadEvent.EventName +EventName = log.OnroadEvent.EventName ButtonType = car.CarState.ButtonEvent.Type SafetyModel = car.CarParams.SafetyModel diff --git a/selfdrive/selfdrived/tests/test_alerts.py b/selfdrive/selfdrived/tests/test_alerts.py index a5249cc03f..b916741ad0 100644 --- a/selfdrive/selfdrived/tests/test_alerts.py +++ b/selfdrive/selfdrived/tests/test_alerts.py @@ -38,7 +38,7 @@ class TestAlerts: def test_events_defined(self): # Ensure all events in capnp schema are defined in events.py - events = car.OnroadEvent.EventName.schema.enumerants + events = log.OnroadEvent.EventName.schema.enumerants for name, e in events.items(): if not name.endswith("DEPRECATED"): diff --git a/selfdrive/test/process_replay/migration.py b/selfdrive/test/process_replay/migration.py index c4f1d19d9d..a90825cbd0 100644 --- a/selfdrive/test/process_replay/migration.py +++ b/selfdrive/test/process_replay/migration.py @@ -3,7 +3,7 @@ from collections.abc import Callable import functools import capnp -from cereal import messaging, car +from cereal import messaging, car, log from opendbc.car.fingerprints import MIGRATION from opendbc.car.toyota.values import EPS_SCALE from openpilot.selfdrive.modeld.constants import ModelConstants @@ -37,6 +37,8 @@ def migrate_all(lr: LogIterable, manager_states: bool = False, panda_states: boo migrate_liveTracks, migrate_driverAssistance, migrate_drivingModelData, + migrate_onroadEvents, + migrate_driverMonitoringState, ] if manager_states: migrations.append(migrate_managerState) @@ -393,3 +395,31 @@ def migrate_sensorEvents(msgs): add_ops.append(m.as_reader()) del_ops.append(index) return [], add_ops, del_ops + + +@migration(inputs=["onroadEventsDEPRECATED"], product="onroadEvents") +def migrate_onroadEvents(msgs): + ops = [] + for index, msg in msgs: + new_msg = messaging.new_message('onroadEvents', len(msg.onroadEventsDEPRECATED)) + new_msg.valid = msg.valid + new_msg.logMonoTime = msg.logMonoTime + + # dict converts name enum into string representation + new_msg.onroadEvents = [log.OnroadEvent(**event.to_dict()) for event in msg.onroadEventsDEPRECATED] + ops.append((index, new_msg.as_reader())) + + return ops, [], [] + + +@migration(inputs=["driverMonitoringState"]) +def migrate_driverMonitoringState(msgs): + ops = [] + for index, msg in msgs: + msg = msg.as_builder() + # dict converts name enum into string representation + msg.driverMonitoringState.events = [log.OnroadEvent(**event.to_dict()) for event in + msg.driverMonitoringState.eventsDEPRECATED] + ops.append((index, msg.as_reader())) + + return ops, [], [] diff --git a/selfdrive/test/process_replay/ref_commit b/selfdrive/test/process_replay/ref_commit index 9aa61ab9c4..eb67f60cc0 100644 --- a/selfdrive/test/process_replay/ref_commit +++ b/selfdrive/test/process_replay/ref_commit @@ -1 +1 @@ -fbf16d1e9b056830a12fcf29a5137e1fc0b01354 \ No newline at end of file +05570b52a90fb8bf092f7a2563d6019577e1aa5d \ No newline at end of file diff --git a/selfdrive/test/test_onroad.py b/selfdrive/test/test_onroad.py index f2afad82ee..f27e444dbe 100644 --- a/selfdrive/test/test_onroad.py +++ b/selfdrive/test/test_onroad.py @@ -13,7 +13,7 @@ from collections import Counter, defaultdict from functools import cached_property from pathlib import Path -from cereal import car +from cereal import log import cereal.messaging as messaging from cereal.services import SERVICE_LIST from openpilot.common.basedir import BASEDIR @@ -422,7 +422,7 @@ class TestOnroad: if msg.which() == "selfdriveState": startup_alert = msg.selfdriveState.alertText1 break - expected = EVENTS[car.OnroadEvent.EventName.startup][ET.PERMANENT].alert_text_1 + expected = EVENTS[log.OnroadEvent.EventName.startup][ET.PERMANENT].alert_text_1 assert startup_alert == expected, "wrong startup alert" def test_engagable(self): diff --git a/selfdrive/test/test_time_to_onroad.py b/selfdrive/test/test_time_to_onroad.py index d3692eabe2..e68386bb58 100644 --- a/selfdrive/test/test_time_to_onroad.py +++ b/selfdrive/test/test_time_to_onroad.py @@ -3,13 +3,13 @@ import pytest import time import subprocess -from cereal import car +from cereal import log import cereal.messaging as messaging from openpilot.common.basedir import BASEDIR from openpilot.common.timeout import Timeout from openpilot.selfdrive.test.helpers import set_params_enabled -EventName = car.OnroadEvent.EventName +EventName = log.OnroadEvent.EventName @pytest.mark.tici