updated: only run offroad & disable automatic git garbage collection (#24269)

* updated: disable automatic git garbage collection

* set gc.autoDetach false

* add gc cloudlogs

* trigger updated

* handle exception

* trigger updated

* only while offroad

* no trigger needed

* trigger updated

* check if git gc was interrupted

* trigger updated

* dont set params when shutting down

* let manager clear params

* no offroad check anymore

* event is nicer to parse

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

selfdrive/common/params.cc

@@ -134,7 +134,7 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"LastPeripheralPandaType", PERSISTENT},
     {"LastPowerDropDetected", CLEAR_ON_MANAGER_START},
     {"LastSystemShutdown", CLEAR_ON_MANAGER_START},
-    {"LastUpdateException", PERSISTENT},
+    {"LastUpdateException", CLEAR_ON_MANAGER_START},
     {"LastUpdateTime", PERSISTENT},
     {"LiveParameters", PERSISTENT},
     {"NavDestination", CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_OFF},

selfdrive/manager/manager.py

@@ -130,7 +130,6 @@ def manager_thread() -> None:
 
   ensure_running(managed_processes.values(), started=False, not_run=ignore)
 
-  started_prev = False
   sm = messaging.SubMaster(['deviceState', 'carParams'], poll=['deviceState'])
   pm = messaging.PubMaster(['managerState'])
 
@@ -141,13 +140,6 @@ def manager_thread() -> None:
     driverview = params.get_bool("IsDriverViewEnabled")
     ensure_running(managed_processes.values(), started=started, driverview=driverview, notcar=sm['carParams'].notCar, not_run=ignore)
 
-    # trigger an update after going offroad
-    if started_prev and not started and 'updated' in managed_processes:
-      os.sync()
-      managed_processes['updated'].signal(signal.SIGHUP)
-
-    started_prev = started
-
     running = ' '.join("%s%s\u001b[0m" % ("\u001b[32m" if p.proc.is_alive() else "\u001b[31m", p.name)
                        for p in managed_processes.values() if p.proc)
     print(running)

selfdrive/manager/process_config.py

@@ -34,7 +34,7 @@ procs = [
   PythonProcess("thermald", "selfdrive.thermald.thermald", offroad=True),
   PythonProcess("timezoned", "selfdrive.timezoned", enabled=TICI, offroad=True),
   PythonProcess("tombstoned", "selfdrive.tombstoned", enabled=not PC, offroad=True),
-  PythonProcess("updated", "selfdrive.updated", enabled=not PC, offroad=True),
+  PythonProcess("updated", "selfdrive.updated", enabled=not PC, onroad=False, offroad=True),
   PythonProcess("uploader", "selfdrive.loggerd.uploader", offroad=True),
   PythonProcess("statsd", "selfdrive.statsd", offroad=True),
 

selfdrive/updated.py

@@ -170,6 +170,8 @@ def setup_git_options(cwd: str) -> None:
     ("core.trustctime", "false"),
     ("core.checkStat", "minimal"),
     ("protocol.version", "2"),
+    ("gc.auto", "0"),
+    ("gc.autoDetach", "false"),
   ]
   for option, value in git_cfg:
     run(["git", "config", option, value], cwd)
@@ -240,7 +242,7 @@ def init_overlay() -> None:
   cloudlog.info(f"git diff output:\n{git_diff}")
 
 
-def finalize_update() -> None:
+def finalize_update(wait_helper: WaitTimeHelper) -> None:
   """Take the current OverlayFS merged view and finalize a copy outside of
   OverlayFS, ready to be swapped-in at BASEDIR. Copy using shutil.copytree"""
 
@@ -256,8 +258,19 @@ def finalize_update() -> None:
   run(["git", "reset", "--hard"], FINALIZED)
   run(["git", "submodule", "foreach", "--recursive", "git", "reset"], FINALIZED)
 
-  set_consistent_flag(True)
-  cloudlog.info("done finalizing overlay")
+  cloudlog.info("Starting git gc")
+  t = time.monotonic()
+  try:
+    run(["git", "gc"], FINALIZED)
+    cloudlog.event("Done git gc", duration=time.monotonic() - t)
+  except subprocess.CalledProcessError:
+    cloudlog.exception(f"Failed git gc, took {time.monotonic() - t:.3f} s")
+
+  if wait_helper.shutdown:
+    cloudlog.info("got interrupted finalizing overlay")
+  else:
+    set_consistent_flag(True)
+    cloudlog.info("done finalizing overlay")
 
 
 def handle_agnos_update(wait_helper: WaitTimeHelper) -> None:
@@ -328,7 +341,7 @@ def fetch_update(wait_helper: WaitTimeHelper) -> bool:
         handle_agnos_update(wait_helper)
 
     # Create the finalized, ready-to-swap update
-    finalize_update()
+    finalize_update(wait_helper)
     cloudlog.info("openpilot update successful!")
   else:
     cloudlog.info("nothing new from git at this time")
@@ -367,10 +380,7 @@ def main() -> None:
 
   first_run = True
   last_fetch_time = 0.0
-  update_failed_count = 0
-
-  # Set initial params for offroad alerts
-  set_params(False, 0, None)
+  update_failed_count = 0  # TODO: Load from param?
 
   # Wait for IsOffroad to be set before our first update attempt
   wait_helper = WaitTimeHelper(proc)
@@ -383,14 +393,6 @@ def main() -> None:
     update_now = wait_helper.ready_event.is_set()
     wait_helper.ready_event.clear()
 
-    # Don't run updater while onroad or if the time's wrong
-    time_wrong = datetime.datetime.utcnow().year < 2019
-    is_onroad = not params.get_bool("IsOffroad")
-    if is_onroad or time_wrong:
-      wait_helper.sleep(30)
-      cloudlog.info("not running updater, not offroad")
-      continue
-
     # Attempt an update
     exception = None
     new_version = False
@@ -425,10 +427,11 @@ def main() -> None:
       exception = str(e)
       overlay_init.unlink(missing_ok=True)
 
-    try:
-      set_params(new_version, update_failed_count, exception)
-    except Exception:
-      cloudlog.exception("uncaught updated exception while setting params, shouldn't happen")
+    if not wait_helper.shutdown:
+      try:
+        set_params(new_version, update_failed_count, exception)
+      except Exception:
+        cloudlog.exception("uncaught updated exception while setting params, shouldn't happen")
 
     # TODO: replace this with a good backoff
     wait_helper.sleep(300)