diff --git a/cereal b/cereal index 3d5e778425..1620ce54ee 160000 --- a/cereal +++ b/cereal @@ -1 +1 @@ -Subproject commit 3d5e7784254ece1b356b47e3327b8e7390b6aa57 +Subproject commit 1620ce54ee878069f68a023ee12c6a9a7d7e29c7 diff --git a/selfdrive/controls/controlsd.py b/selfdrive/controls/controlsd.py index 54db6976d6..198ee08b3e 100755 --- a/selfdrive/controls/controlsd.py +++ b/selfdrive/controls/controlsd.py @@ -293,12 +293,20 @@ class Controls: self.events.add(EventName.radarFault) elif not self.sm.valid["pandaStates"]: self.events.add(EventName.usbError) - elif not self.sm.all_alive_and_valid() or self.can_rcv_error: - self.events.add(EventName.commIssue) + elif not self.sm.all_checks() or self.can_rcv_error: + + if not self.sm.all_alive(): + self.events.add(EventName.commIssue) + elif not self.sm.all_freq_ok(): + self.events.add(EventName.commIssueAvgFreq) + else: # invalid or can_rcv_error. + self.events.add(EventName.commIssue) + if not self.logged_comm_issue: invalid = [s for s, valid in self.sm.valid.items() if not valid] not_alive = [s for s, alive in self.sm.alive.items() if not alive] - cloudlog.event("commIssue", invalid=invalid, not_alive=not_alive, can_error=self.can_rcv_error, error=True) + not_freq_ok = [s for s, freq_ok in self.sm.freq_ok.items() if not freq_ok] + cloudlog.event("commIssue", invalid=invalid, not_alive=not_alive, not_freq_ok=not_freq_ok, can_error=self.can_rcv_error, error=True) self.logged_comm_issue = True else: self.logged_comm_issue = False @@ -347,8 +355,13 @@ class Controls: if not self.sm['liveLocationKalman'].gpsOK and (self.distance_traveled > 1000): # Not show in first 1 km to allow for driving out of garage. This event shows after 5 minutes self.events.add(EventName.noGps) + + # TODO: split into separate alerts if not self.sm.all_alive(self.camera_packets): self.events.add(EventName.cameraMalfunction) + elif not self.sm.all_freq_ok(self.camera_packets): + self.events.add(EventName.cameraMalfunction) + if self.sm['modelV2'].frameDropPerc > 20: self.events.add(EventName.modeldLagging) if self.sm['liveLocationKalman'].excessiveResets: @@ -379,7 +392,7 @@ class Controls: self.sm.update(0) if not self.initialized: - all_valid = CS.canValid and self.sm.all_alive_and_valid() + all_valid = CS.canValid and self.sm.all_checks() if all_valid or self.sm.frame * DT_CTRL > 3.5 or SIMULATION: if not self.read_only: self.CI.init(self.CP, self.can_sock, self.pm.sock['sendcan']) diff --git a/selfdrive/controls/lib/events.py b/selfdrive/controls/lib/events.py index 7919c40206..54ca53d7ec 100644 --- a/selfdrive/controls/lib/events.py +++ b/selfdrive/controls/lib/events.py @@ -660,6 +660,10 @@ EVENTS: Dict[int, Dict[str, Union[Alert, AlertCallbackType]]] = { ET.SOFT_DISABLE: soft_disable_alert("Communication Issue between Processes"), ET.NO_ENTRY: NoEntryAlert("Communication Issue between Processes"), }, + EventName.commIssueAvgFreq: { + ET.SOFT_DISABLE: soft_disable_alert("Low Communication Rate between Processes"), + ET.NO_ENTRY: NoEntryAlert("Low Communication Rate between Processes"), + }, # Thrown when manager detects a service exited unexpectedly while driving EventName.processNotRunning: { diff --git a/selfdrive/controls/lib/lateral_planner.py b/selfdrive/controls/lib/lateral_planner.py index 2590465713..f60b3159b9 100644 --- a/selfdrive/controls/lib/lateral_planner.py +++ b/selfdrive/controls/lib/lateral_planner.py @@ -100,7 +100,7 @@ class LateralPlanner: def publish(self, sm, pm): plan_solution_valid = self.solution_invalid_cnt < 2 plan_send = messaging.new_message('lateralPlan') - plan_send.valid = sm.all_alive_and_valid(service_list=['carState', 'controlsState', 'modelV2']) + plan_send.valid = sm.all_checks(service_list=['carState', 'controlsState', 'modelV2']) lateralPlan = plan_send.lateralPlan lateralPlan.modelMonoTime = sm.logMonoTime['modelV2'] diff --git a/selfdrive/controls/lib/longitudinal_planner.py b/selfdrive/controls/lib/longitudinal_planner.py index 41c41e5493..0d21c519a3 100755 --- a/selfdrive/controls/lib/longitudinal_planner.py +++ b/selfdrive/controls/lib/longitudinal_planner.py @@ -114,7 +114,7 @@ class Planner: def publish(self, sm, pm): plan_send = messaging.new_message('longitudinalPlan') - plan_send.valid = sm.all_alive_and_valid(service_list=['carState', 'controlsState']) + plan_send.valid = sm.all_checks(service_list=['carState', 'controlsState']) longitudinalPlan = plan_send.longitudinalPlan longitudinalPlan.modelMonoTime = sm.logMonoTime['modelV2'] diff --git a/selfdrive/controls/radard.py b/selfdrive/controls/radard.py index ddba0920b8..cae9474faf 100755 --- a/selfdrive/controls/radard.py +++ b/selfdrive/controls/radard.py @@ -163,7 +163,7 @@ class RadarD(): # *** publish radarState *** dat = messaging.new_message('radarState') - dat.valid = sm.all_alive_and_valid() and len(rr.errors) == 0 + dat.valid = sm.all_checks() and len(rr.errors) == 0 radarState = dat.radarState radarState.mdMonoTime = sm.logMonoTime['modelV2'] radarState.canMonoTimes = list(rr.canMonoTimes) diff --git a/selfdrive/locationd/paramsd.py b/selfdrive/locationd/paramsd.py index 44ffccdf1a..55479f255c 100755 --- a/selfdrive/locationd/paramsd.py +++ b/selfdrive/locationd/paramsd.py @@ -160,7 +160,7 @@ def main(sm=None, pm=None): while True: sm.update() - if sm.all_alive_and_valid(): + if sm.all_checks(): for which in sorted(sm.updated.keys(), key=lambda x: sm.logMonoTime[x]): if sm.updated[which]: t = sm.logMonoTime[which] * 1e-9 @@ -199,7 +199,7 @@ def main(sm=None, pm=None): liveParameters.angleOffsetAverageStd = float(P[States.ANGLE_OFFSET]) liveParameters.angleOffsetFastStd = float(P[States.ANGLE_OFFSET_FAST]) - msg.valid = sm.all_alive_and_valid() + msg.valid = sm.all_checks() if sm.frame % 1200 == 0: # once a minute params = { diff --git a/selfdrive/test/process_replay/ref_commit b/selfdrive/test/process_replay/ref_commit index fa06c2891f..0bf43aa887 100644 --- a/selfdrive/test/process_replay/ref_commit +++ b/selfdrive/test/process_replay/ref_commit @@ -1 +1 @@ -d71efcf28d2216aa53eb4f272514d28c9f96d433 \ No newline at end of file +c1a729c087e9f26e40d8f7db5bdfe62d12fb0bc9 \ No newline at end of file diff --git a/selfdrive/test/profiling/lib.py b/selfdrive/test/profiling/lib.py index fcc832d327..f28346f3f0 100644 --- a/selfdrive/test/profiling/lib.py +++ b/selfdrive/test/profiling/lib.py @@ -44,6 +44,7 @@ class SubMaster(messaging.SubMaster): self.rcv_time = {s: 0. for s in services} self.rcv_frame = {s: 0 for s in services} self.valid = {s: True for s in services} + self.freq_ok = {s: True for s in services} self.recv_dts = {s: deque([0.0] * messaging.AVG_FREQ_HISTORY, maxlen=messaging.AVG_FREQ_HISTORY) for s in services} self.logMonoTime = {} self.sock = {}