From 1cc4f1a6341b58b1541f78c62e3b901c601f52bd Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 3 Dec 2023 10:50:17 -0800 Subject: [PATCH] new msgs default to invalid (#30587) * new msgs default to invalid * fix lm * set more valid * update rest * update refs * fix logMessage * more valids * cleanup * fix llk test * pigeond is also valid * more valids old-commit-hash: a1d36961cf9fa0c6a9c41fe92d6504de5d36cbaa --- cereal | 2 +- selfdrive/controls/controlsd.py | 4 +++- selfdrive/controls/radard.py | 1 + selfdrive/locationd/calibrationd.py | 26 +++++++++------------- selfdrive/locationd/test/test_locationd.py | 1 + selfdrive/manager/manager.py | 2 +- selfdrive/modeld/dmonitoringmodeld.py | 2 +- selfdrive/monitoring/dmonitoringd.py | 2 +- selfdrive/navd/navd.py | 4 ++-- selfdrive/test/process_replay/migration.py | 2 ++ selfdrive/test/process_replay/ref_commit | 2 +- selfdrive/thermald/thermald.py | 2 +- system/loggerd/uploader.py | 2 +- system/logmessaged.py | 6 ++--- system/micd.py | 2 +- system/qcomgpsd/qcomgpsd.py | 8 +++---- system/sensord/pigeond.py | 2 +- 17 files changed, 34 insertions(+), 36 deletions(-) diff --git a/cereal b/cereal index 60d1ee0490..033dad2b37 160000 --- a/cereal +++ b/cereal @@ -1 +1 @@ -Subproject commit 60d1ee0490a33830168b30a1e92f8d4e43cfbb15 +Subproject commit 033dad2b370dac411479de19381f6e0253e51e01 diff --git a/selfdrive/controls/controlsd.py b/selfdrive/controls/controlsd.py index a04349cd67..ed4c912608 100755 --- a/selfdrive/controls/controlsd.py +++ b/selfdrive/controls/controlsd.py @@ -80,7 +80,7 @@ class Controls: 'driverMonitoringState', 'longitudinalPlan', 'lateralPlan', 'liveLocationKalman', 'managerState', 'liveParameters', 'radarState', 'liveTorqueParameters', 'testJoystick'] + self.camera_packets + self.sensor_packets, - ignore_alive=ignore, ignore_avg_freq=['radarState', 'testJoystick']) + ignore_alive=ignore, ignore_avg_freq=['radarState', 'testJoystick'], ignore_valid=['testJoystick', ]) if CI is None: # wait for one pandaState and one CAN packet @@ -824,6 +824,7 @@ class Controls: # onroadEvents - logged every second or on change if (self.sm.frame % int(1. / DT_CTRL) == 0) or (self.events.names != self.events_prev): ce_send = messaging.new_message('onroadEvents', len(self.events)) + ce_send.valid = True ce_send.onroadEvents = car_events self.pm.send('onroadEvents', ce_send) self.events_prev = self.events.names.copy() @@ -831,6 +832,7 @@ class Controls: # carParams - logged every 50 seconds (> 1 per segment) if (self.sm.frame % int(50. / DT_CTRL) == 0): cp_send = messaging.new_message('carParams') + cp_send.valid = True cp_send.carParams = self.CP self.pm.send('carParams', cp_send) diff --git a/selfdrive/controls/radard.py b/selfdrive/controls/radard.py index ec42ae66f2..a2d58cfed2 100755 --- a/selfdrive/controls/radard.py +++ b/selfdrive/controls/radard.py @@ -270,6 +270,7 @@ class RadarD: # publish tracks for UI debugging (keep last) tracks_msg = messaging.new_message('liveTracks', len(self.tracks)) + tracks_msg.valid = self.radar_state_valid for index, tid in enumerate(sorted(self.tracks.keys())): tracks_msg.liveTracks[index] = { "trackId": tid, diff --git a/selfdrive/locationd/calibrationd.py b/selfdrive/locationd/calibrationd.py index 566373d7b8..654c72528b 100755 --- a/selfdrive/locationd/calibrationd.py +++ b/selfdrive/locationd/calibrationd.py @@ -164,7 +164,7 @@ class Calibrator: write_this_cycle = (self.idx == 0) and (self.block_idx % (INPUTS_WANTED//5) == 5) if self.param_put and write_this_cycle: - put_nonblocking("CalibrationParams", self.get_msg().to_bytes()) + put_nonblocking("CalibrationParams", self.get_msg(True).to_bytes()) def handle_v_ego(self, v_ego: float) -> None: self.v_ego = v_ego @@ -227,12 +227,13 @@ class Calibrator: return new_rpy - def get_msg(self) -> capnp.lib.capnp._DynamicStructBuilder: + def get_msg(self, valid: bool) -> capnp.lib.capnp._DynamicStructBuilder: smooth_rpy = self.get_smooth_rpy() msg = messaging.new_message('liveCalibration') - liveCalibration = msg.liveCalibration + msg.valid = valid + liveCalibration = msg.liveCalibration liveCalibration.validBlocks = self.valid_blocks liveCalibration.calStatus = self.cal_status liveCalibration.calPerc = min(100 * (self.valid_blocks * BLOCK_SIZE + self.idx) // (INPUTS_NEEDED * BLOCK_SIZE), 100) @@ -250,19 +251,16 @@ class Calibrator: return msg - def send_data(self, pm: messaging.PubMaster) -> None: - pm.send('liveCalibration', self.get_msg()) + def send_data(self, pm: messaging.PubMaster, valid: bool) -> None: + pm.send('liveCalibration', self.get_msg(valid)) -def calibrationd_thread(sm: Optional[messaging.SubMaster] = None, pm: Optional[messaging.PubMaster] = None) -> NoReturn: +def main() -> NoReturn: gc.disable() set_realtime_priority(1) - if sm is None: - sm = messaging.SubMaster(['cameraOdometry', 'carState', 'carParams'], poll=['cameraOdometry']) - - if pm is None: - pm = messaging.PubMaster(['liveCalibration']) + pm = messaging.PubMaster(['liveCalibration']) + sm = messaging.SubMaster(['cameraOdometry', 'carState', 'carParams'], poll=['cameraOdometry']) calibrator = Calibrator(param_put=True) @@ -286,11 +284,7 @@ def calibrationd_thread(sm: Optional[messaging.SubMaster] = None, pm: Optional[m # 4Hz driven by cameraOdometry if sm.frame % 5 == 0: - calibrator.send_data(pm) - - -def main(sm: Optional[messaging.SubMaster] = None, pm: Optional[messaging.PubMaster] = None) -> NoReturn: - calibrationd_thread(sm, pm) + calibrator.send_data(pm, sm.all_checks()) if __name__ == "__main__": diff --git a/selfdrive/locationd/test/test_locationd.py b/selfdrive/locationd/test/test_locationd.py index 707fdd743f..78de9216dc 100755 --- a/selfdrive/locationd/test/test_locationd.py +++ b/selfdrive/locationd/test/test_locationd.py @@ -60,6 +60,7 @@ class TestLocationdProc(unittest.TestCase): msg.cameraOdometry.trans = [0.0, 0.0, 0.0] msg.cameraOdometry.transStd = [0.0, 0.0, 0.0] msg.logMonoTime = t + msg.valid = True return msg def test_params_gps(self): diff --git a/selfdrive/manager/manager.py b/selfdrive/manager/manager.py index 9be448499e..7b23445c8b 100755 --- a/selfdrive/manager/manager.py +++ b/selfdrive/manager/manager.py @@ -171,7 +171,7 @@ def manager_thread() -> None: cloudlog.debug(running) # send managerState - msg = messaging.new_message('managerState') + msg = messaging.new_message('managerState', valid=True) msg.managerState.processes = [p.get_process_state_msg() for p in managed_processes.values()] pm.send('managerState', msg) diff --git a/selfdrive/modeld/dmonitoringmodeld.py b/selfdrive/modeld/dmonitoringmodeld.py index 53c0af0ff3..aafe78abde 100755 --- a/selfdrive/modeld/dmonitoringmodeld.py +++ b/selfdrive/modeld/dmonitoringmodeld.py @@ -101,7 +101,7 @@ def fill_driver_state(msg, ds_result: DriverStateResult): def get_driverstate_packet(model_output: np.ndarray, frame_id: int, location_ts: int, execution_time: float, dsp_execution_time: float): model_result = ctypes.cast(model_output.ctypes.data, ctypes.POINTER(DMonitoringModelResult)).contents - msg = messaging.new_message('driverStateV2') + msg = messaging.new_message('driverStateV2', valid=True) ds = msg.driverStateV2 ds.frameId = frame_id ds.modelExecutionTime = execution_time diff --git a/selfdrive/monitoring/dmonitoringd.py b/selfdrive/monitoring/dmonitoringd.py index 6ea35a6c63..7455459967 100755 --- a/selfdrive/monitoring/dmonitoringd.py +++ b/selfdrive/monitoring/dmonitoringd.py @@ -54,7 +54,7 @@ def dmonitoringd_thread(): driver_status.update_events(events, driver_engaged, sm['controlsState'].enabled, sm['carState'].standstill) # build driverMonitoringState packet - dat = messaging.new_message('driverMonitoringState') + dat = messaging.new_message('driverMonitoringState', valid=sm.all_checks()) dat.driverMonitoringState = { "events": events.to_msg(), "faceDetected": driver_status.face_detected, diff --git a/selfdrive/navd/navd.py b/selfdrive/navd/navd.py index da2b8c06b9..0d71efcc8d 100755 --- a/selfdrive/navd/navd.py +++ b/selfdrive/navd/navd.py @@ -196,7 +196,7 @@ class RouteEngine: self.send_route() def send_instruction(self): - msg = messaging.new_message('navInstruction') + msg = messaging.new_message('navInstruction', valid=True) if self.step_idx is None: msg.valid = False @@ -302,7 +302,7 @@ class RouteEngine: for path in self.route_geometry: coords += [c.as_dict() for c in path] - msg = messaging.new_message('navRoute') + msg = messaging.new_message('navRoute', valid=True) msg.navRoute.coordinates = coords self.pm.send('navRoute', msg) diff --git a/selfdrive/test/process_replay/migration.py b/selfdrive/test/process_replay/migration.py index ad5192dbba..831bea7c6f 100644 --- a/selfdrive/test/process_replay/migration.py +++ b/selfdrive/test/process_replay/migration.py @@ -85,6 +85,7 @@ def migrate_peripheralState(lr): continue new_msg = messaging.new_message("peripheralState") + new_msg.valid = msg.valid new_msg.logMonoTime = msg.logMonoTime all_msg.append(new_msg.as_reader()) @@ -149,6 +150,7 @@ def migrate_carParams(lr, old_logtime=False): for msg in lr: if msg.which() == 'carParams': CP = messaging.new_message('carParams') + CP.valid = True CP.carParams = msg.carParams.as_builder() for car_fw in CP.carParams.carFw: car_fw.brand = CP.carParams.carName diff --git a/selfdrive/test/process_replay/ref_commit b/selfdrive/test/process_replay/ref_commit index 2ab22ca9ef..030cbc4e07 100644 --- a/selfdrive/test/process_replay/ref_commit +++ b/selfdrive/test/process_replay/ref_commit @@ -1 +1 @@ -a42195085f2b92df02d13d60a8cee80354a84c7a \ No newline at end of file +921222d49db204071f0a7006fc895690e1045b5d \ No newline at end of file diff --git a/selfdrive/thermald/thermald.py b/selfdrive/thermald/thermald.py index 3ccb9349e1..690618d0b8 100755 --- a/selfdrive/thermald/thermald.py +++ b/selfdrive/thermald/thermald.py @@ -81,7 +81,7 @@ def read_tz(x): def read_thermal(thermal_config): - dat = messaging.new_message('deviceState') + dat = messaging.new_message('deviceState', valid=True) dat.deviceState.cpuTempC = [read_tz(z) / thermal_config.cpu[1] for z in thermal_config.cpu[0]] dat.deviceState.gpuTempC = [read_tz(z) / thermal_config.gpu[1] for z in thermal_config.gpu[0]] dat.deviceState.memoryTempC = read_tz(thermal_config.mem[0]) / thermal_config.mem[1] diff --git a/system/loggerd/uploader.py b/system/loggerd/uploader.py index 12c6ecdca9..b5804b1624 100755 --- a/system/loggerd/uploader.py +++ b/system/loggerd/uploader.py @@ -228,7 +228,7 @@ class Uploader: return success def get_msg(self): - msg = messaging.new_message("uploaderState") + msg = messaging.new_message("uploaderState", valid=True) us = msg.uploaderState us.immediateQueueSize = int(self.immediate_size / 1e6) us.immediateQueueCount = self.immediate_count diff --git a/system/logmessaged.py b/system/logmessaged.py index c53e20e483..5bd83f4b05 100755 --- a/system/logmessaged.py +++ b/system/logmessaged.py @@ -35,13 +35,11 @@ def main() -> NoReturn: continue # then we publish them - msg = messaging.new_message() - msg.logMessage = record + msg = messaging.new_message(None, valid=True, logMessage=record) log_message_sock.send(msg.to_bytes()) if level >= 40: # logging.ERROR - msg = messaging.new_message() - msg.errorLogMessage = record + msg = messaging.new_message(None, valid=True, errorLogMessage=record) error_log_message_sock.send(msg.to_bytes()) finally: sock.close() diff --git a/system/micd.py b/system/micd.py index 72f3b8b490..0b7a4dadfb 100755 --- a/system/micd.py +++ b/system/micd.py @@ -53,7 +53,7 @@ class Mic: self.spl_filter_weighted = FirstOrderFilter(0, 2.5, FILTER_DT, initialized=False) def update(self): - msg = messaging.new_message('microphone') + msg = messaging.new_message('microphone', valid=True) msg.microphone.soundPressure = float(self.sound_pressure) msg.microphone.soundPressureWeighted = float(self.sound_pressure_weighted) diff --git a/system/qcomgpsd/qcomgpsd.py b/system/qcomgpsd/qcomgpsd.py index ac862052d3..78539266c0 100755 --- a/system/qcomgpsd/qcomgpsd.py +++ b/system/qcomgpsd/qcomgpsd.py @@ -321,7 +321,7 @@ def main() -> NoReturn: print("%.4f: got log: %x len %d" % (time.time(), log_type, len(log_payload))) if log_type == LOG_GNSS_OEMDRE_MEASUREMENT_REPORT: - msg = messaging.new_message('qcomGnss') + msg = messaging.new_message('qcomGnss', valid=True) gnss = msg.qcomGnss gnss.logTs = log_time @@ -370,7 +370,7 @@ def main() -> NoReturn: vNED = [report["q_FltVelEnuMps[1]"], report["q_FltVelEnuMps[0]"], -report["q_FltVelEnuMps[2]"]] vNEDsigma = [report["q_FltVelSigmaMps[1]"], report["q_FltVelSigmaMps[0]"], -report["q_FltVelSigmaMps[2]"]] - msg = messaging.new_message('gpsLocation') + msg = messaging.new_message('gpsLocation', valid=True) gps = msg.gpsLocation gps.latitude = report["t_DblFinalPosLatLon[0]"] * 180/math.pi gps.longitude = report["t_DblFinalPosLatLon[1]"] * 180/math.pi @@ -396,7 +396,7 @@ def main() -> NoReturn: pm.send('gpsLocation', msg) elif log_type == LOG_GNSS_OEMDRE_SVPOLY_REPORT: - msg = messaging.new_message('qcomGnss') + msg = messaging.new_message('qcomGnss', valid=True) dat = unpack_svpoly(log_payload) dat = relist(dat) gnss = msg.qcomGnss @@ -433,7 +433,7 @@ def main() -> NoReturn: pm.send('qcomGnss', msg) elif log_type in [LOG_GNSS_GPS_MEASUREMENT_REPORT, LOG_GNSS_GLONASS_MEASUREMENT_REPORT]: - msg = messaging.new_message('qcomGnss') + msg = messaging.new_message('qcomGnss', valid=True) gnss = msg.qcomGnss gnss.logTs = log_time diff --git a/system/sensord/pigeond.py b/system/sensord/pigeond.py index 2e8f151d17..6a02f627b9 100755 --- a/system/sensord/pigeond.py +++ b/system/sensord/pigeond.py @@ -291,7 +291,7 @@ def run_receiving(pigeon: TTYPigeon, pm: messaging.PubMaster, duration: int = 0) continue # send out to socket - msg = messaging.new_message('ubloxRaw', len(dat)) + msg = messaging.new_message('ubloxRaw', len(dat), valid=True) msg.ubloxRaw = dat[:] pm.send('ubloxRaw', msg) else: