tici: restart ModemManager if probing fails (#29114)

* tici: restart ModemManager if probing fails

* add comment

---------

Co-authored-by: Comma Device <device@comma.ai>
old-commit-hash: 787d3b1714

selfdrive/thermald/thermald.py

@@ -103,6 +103,8 @@ def hw_state_thread(end_event, hw_queue):
   modem_version = None
   modem_nv = None
   modem_configured = False
+  modem_restarted = False
+  modem_missing_count = 0
 
   while not end_event.is_set():
     # these are expensive calls. update every 10s
@@ -120,6 +122,16 @@ def hw_state_thread(end_event, hw_queue):
 
           if (modem_version is not None) and (modem_nv is not None):
             cloudlog.event("modem version", version=modem_version, nv=modem_nv)
+          else:
+            if not modem_restarted:
+              # TODO: we may be able to remove this with a MM update
+              # ModemManager's probing on startup can fail
+              # rarely, restart the service to probe again.
+              modem_missing_count += 1
+              if modem_missing_count > 3:
+                modem_restarted = True
+                cloudlog.event("restarting ModemManager")
+                os.system("sudo systemctl restart --no-block ModemManager")
 
         tx, rx = HARDWARE.get_modem_data_usage()