commIssueAvgFreq: separate alert for avg freq check (#24150)

* commIssueAvgFreq: separate alert for avg freq check

* bump cereal

* bump cereal

* bump cereal

* freq_ok separate from alive

* bump cereal to master

* update_ref

* fix profiler

cereal

@@ -1 +1 @@
-Subproject commit 3d5e7784254ece1b356b47e3327b8e7390b6aa57
+Subproject commit 1620ce54ee878069f68a023ee12c6a9a7d7e29c7

selfdrive/controls/controlsd.py

@@ -293,12 +293,20 @@ class Controls:
       self.events.add(EventName.radarFault)
     elif not self.sm.valid["pandaStates"]:
       self.events.add(EventName.usbError)
-    elif not self.sm.all_alive_and_valid() or self.can_rcv_error:
+    elif not self.sm.all_checks() or self.can_rcv_error:
+
+      if not self.sm.all_alive():
+        self.events.add(EventName.commIssue)
+      elif not self.sm.all_freq_ok():
+        self.events.add(EventName.commIssueAvgFreq)
+      else: # invalid or can_rcv_error.
         self.events.add(EventName.commIssue)
+
       if not self.logged_comm_issue:
         invalid = [s for s, valid in self.sm.valid.items() if not valid]
         not_alive = [s for s, alive in self.sm.alive.items() if not alive]
-        cloudlog.event("commIssue", invalid=invalid, not_alive=not_alive, can_error=self.can_rcv_error, error=True)
+        not_freq_ok = [s for s, freq_ok in self.sm.freq_ok.items() if not freq_ok]
+        cloudlog.event("commIssue", invalid=invalid, not_alive=not_alive, not_freq_ok=not_freq_ok, can_error=self.can_rcv_error, error=True)
         self.logged_comm_issue = True
     else:
       self.logged_comm_issue = False
@@ -347,8 +355,13 @@ class Controls:
         if not self.sm['liveLocationKalman'].gpsOK and (self.distance_traveled > 1000):
           # Not show in first 1 km to allow for driving out of garage. This event shows after 5 minutes
           self.events.add(EventName.noGps)
+
+      # TODO: split into separate alerts
       if not self.sm.all_alive(self.camera_packets):
         self.events.add(EventName.cameraMalfunction)
+      elif not self.sm.all_freq_ok(self.camera_packets):
+        self.events.add(EventName.cameraMalfunction)
+
       if self.sm['modelV2'].frameDropPerc > 20:
         self.events.add(EventName.modeldLagging)
       if self.sm['liveLocationKalman'].excessiveResets:
@@ -379,7 +392,7 @@ class Controls:
     self.sm.update(0)
 
     if not self.initialized:
-      all_valid = CS.canValid and self.sm.all_alive_and_valid()
+      all_valid = CS.canValid and self.sm.all_checks()
       if all_valid or self.sm.frame * DT_CTRL > 3.5 or SIMULATION:
         if not self.read_only:
           self.CI.init(self.CP, self.can_sock, self.pm.sock['sendcan'])

selfdrive/controls/lib/events.py

@@ -660,6 +660,10 @@ EVENTS: Dict[int, Dict[str, Union[Alert, AlertCallbackType]]] = {
     ET.SOFT_DISABLE: soft_disable_alert("Communication Issue between Processes"),
     ET.NO_ENTRY: NoEntryAlert("Communication Issue between Processes"),
   },
+  EventName.commIssueAvgFreq: {
+    ET.SOFT_DISABLE: soft_disable_alert("Low Communication Rate between Processes"),
+    ET.NO_ENTRY: NoEntryAlert("Low Communication Rate between Processes"),
+  },
 
   # Thrown when manager detects a service exited unexpectedly while driving
   EventName.processNotRunning: {

selfdrive/controls/lib/lateral_planner.py

@@ -100,7 +100,7 @@ class LateralPlanner:
   def publish(self, sm, pm):
     plan_solution_valid = self.solution_invalid_cnt < 2
     plan_send = messaging.new_message('lateralPlan')
-    plan_send.valid = sm.all_alive_and_valid(service_list=['carState', 'controlsState', 'modelV2'])
+    plan_send.valid = sm.all_checks(service_list=['carState', 'controlsState', 'modelV2'])
 
     lateralPlan = plan_send.lateralPlan
     lateralPlan.modelMonoTime = sm.logMonoTime['modelV2']

selfdrive/controls/lib/longitudinal_planner.py

@@ -114,7 +114,7 @@ class Planner:
   def publish(self, sm, pm):
     plan_send = messaging.new_message('longitudinalPlan')
 
-    plan_send.valid = sm.all_alive_and_valid(service_list=['carState', 'controlsState'])
+    plan_send.valid = sm.all_checks(service_list=['carState', 'controlsState'])
 
     longitudinalPlan = plan_send.longitudinalPlan
     longitudinalPlan.modelMonoTime = sm.logMonoTime['modelV2']

selfdrive/controls/radard.py

@@ -163,7 +163,7 @@ class RadarD():
 
     # *** publish radarState ***
     dat = messaging.new_message('radarState')
-    dat.valid = sm.all_alive_and_valid() and len(rr.errors) == 0
+    dat.valid = sm.all_checks() and len(rr.errors) == 0
     radarState = dat.radarState
     radarState.mdMonoTime = sm.logMonoTime['modelV2']
     radarState.canMonoTimes = list(rr.canMonoTimes)

selfdrive/locationd/paramsd.py

@@ -160,7 +160,7 @@ def main(sm=None, pm=None):
 
   while True:
     sm.update()
-    if sm.all_alive_and_valid():
+    if sm.all_checks():
       for which in sorted(sm.updated.keys(), key=lambda x: sm.logMonoTime[x]):
         if sm.updated[which]:
           t = sm.logMonoTime[which] * 1e-9
@@ -199,7 +199,7 @@ def main(sm=None, pm=None):
       liveParameters.angleOffsetAverageStd = float(P[States.ANGLE_OFFSET])
       liveParameters.angleOffsetFastStd = float(P[States.ANGLE_OFFSET_FAST])
 
-      msg.valid = sm.all_alive_and_valid()
+      msg.valid = sm.all_checks()
 
       if sm.frame % 1200 == 0:  # once a minute
         params = {

selfdrive/test/process_replay/ref_commit

@@ -1 +1 @@
-d71efcf28d2216aa53eb4f272514d28c9f96d433
+c1a729c087e9f26e40d8f7db5bdfe62d12fb0bc9

selfdrive/test/profiling/lib.py

@@ -44,6 +44,7 @@ class SubMaster(messaging.SubMaster):
     self.rcv_time = {s: 0. for s in services}
     self.rcv_frame = {s: 0 for s in services}
     self.valid = {s: True for s in services}
+    self.freq_ok = {s: True for s in services}
     self.recv_dts = {s: deque([0.0] * messaging.AVG_FREQ_HISTORY, maxlen=messaging.AVG_FREQ_HISTORY) for s in services}
     self.logMonoTime = {}
     self.sock = {}