sentry fixups (#23249)

* set env

* do init in same place

* no sentry for forks

* start session

* start session

* remove that

* val
old-commit-hash: b63bc89bc9

release/files_common

@@ -70,7 +70,7 @@ selfdrive/version.py
 
 selfdrive/__init__.py
 selfdrive/config.py
-selfdrive/crash.py
+selfdrive/sentry.py
 selfdrive/swaglog.py
 selfdrive/logmessaged.py
 selfdrive/tombstoned.py

selfdrive/crash.py

@@ -1,27 +0,0 @@
-"""Install exception handler for process crash."""
-from selfdrive.swaglog import cloudlog
-from selfdrive.version import get_version
-
-import sentry_sdk
-from sentry_sdk.integrations.threading import ThreadingIntegration
-
-def capture_exception(*args, **kwargs) -> None:
-  cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1))
-
-  try:
-    sentry_sdk.capture_exception(*args, **kwargs)
-    sentry_sdk.flush()  # https://github.com/getsentry/sentry-python/issues/291
-  except Exception:
-    cloudlog.exception("sentry exception")
-
-def bind_user(**kwargs) -> None:
-  sentry_sdk.set_user(kwargs)
-
-def bind_extra(**kwargs) -> None:
-  for k, v in kwargs.items():
-    sentry_sdk.set_tag(k, v)
-
-def init() -> None:
-  sentry_sdk.init("https://a8dc76b5bfb34908a601d67e2aa8bcf9@o33823.ingest.sentry.io/77924",
-                  default_integrations=False, integrations=[ThreadingIntegration(propagate_hub=True)],
-                  release=get_version())

selfdrive/manager/manager.py

@@ -8,7 +8,7 @@ import traceback
 from typing import List, Tuple, Union
 
 import cereal.messaging as messaging
-import selfdrive.crash as crash
+import selfdrive.sentry as sentry
 from common.basedir import BASEDIR
 from common.params import Params, ParamKeyType
 from common.text_window import TextWindow
@@ -20,7 +20,7 @@ from selfdrive.manager.process_config import managed_processes
 from selfdrive.athena.registration import register, UNREGISTERED_DONGLE_ID
 from selfdrive.swaglog import cloudlog, add_file_handler
 from selfdrive.version import is_dirty, get_commit, get_version, get_origin, get_short_branch, \
-                              terms_version, training_version, is_comma_remote
+                              terms_version, training_version
 
 
 sys.path.append(os.path.join(BASEDIR, "pyextra"))
@@ -90,15 +90,11 @@ def manager_init() -> None:
   if not is_dirty():
     os.environ['CLEAN'] = '1'
 
+  # init logging
+  sentry.init(sentry.SentryProject.SELFDRIVE)
   cloudlog.bind_global(dongle_id=dongle_id, version=get_version(), dirty=is_dirty(),
                        device=HARDWARE.get_device_type())
 
-  if is_comma_remote() and not (os.getenv("NOLOG") or os.getenv("NOCRASH") or PC):
-    crash.init()
-  crash.bind_user(id=dongle_id)
-  crash.bind_extra(dirty=is_dirty(), origin=get_origin(), branch=get_short_branch(), commit=get_commit(),
-                   device=HARDWARE.get_device_type())
-
 
 def manager_prepare() -> None:
   for p in managed_processes.values():
@@ -195,7 +191,7 @@ def main() -> None:
     manager_thread()
   except Exception:
     traceback.print_exc()
-    crash.capture_exception()
+    sentry.capture_exception()
   finally:
     manager_cleanup()
 

selfdrive/manager/process.py

@@ -11,7 +11,7 @@ from multiprocessing import Process
 from setproctitle import setproctitle  # pylint: disable=no-name-in-module
 
 import cereal.messaging as messaging
-import selfdrive.crash as crash
+import selfdrive.sentry as sentry
 from common.basedir import BASEDIR
 from common.params import Params
 from common.realtime import sec_since_boot
@@ -44,7 +44,7 @@ def launcher(proc: str, name: str) -> None:
   except Exception:
     # can't install the crash handler because sys.excepthook doesn't play nice
     # with threads, so catch it here.
-    crash.capture_exception()
+    sentry.capture_exception()
     raise
 
 

selfdrive/sentry.py

@@ -0,0 +1,69 @@
+"""Install exception handler for process crash."""
+import sentry_sdk
+from enum import Enum
+from sentry_sdk.integrations.threading import ThreadingIntegration
+
+from common.params import Params
+from selfdrive.hardware import HARDWARE
+from selfdrive.swaglog import cloudlog
+from selfdrive.version import get_branch, get_commit, get_origin, get_version, \
+                              is_comma_remote, is_dirty, is_tested_branch
+
+
+class SentryProject(Enum):
+  # python project
+  SELFDRIVE = "https://a8dc76b5bfb34908a601d67e2aa8bcf9@o33823.ingest.sentry.io/77924"
+  # native project
+  SELFDRIVE_NATIVE = "https://a40f22e13cbc4261873333c125fc9d38@o33823.ingest.sentry.io/157615"
+
+
+def report_tombstone(fn: str, message: str, contents: str) -> None:
+  cloudlog.error({'tombstone': message})
+
+  with sentry_sdk.configure_scope() as scope:
+    scope.set_extra("tombstone_fn", fn)
+    scope.set_extra("tombstone", contents)
+    sentry_sdk.capture_message(message=message)
+    sentry_sdk.flush()
+
+
+def capture_exception(*args, **kwargs) -> None:
+  cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1))
+
+  try:
+    sentry_sdk.capture_exception(*args, **kwargs)
+    sentry_sdk.flush()  # https://github.com/getsentry/sentry-python/issues/291
+  except Exception:
+    cloudlog.exception("sentry exception")
+
+
+def init(project: SentryProject) -> None:
+  # forks like to mess with this, so double check
+  if not (is_comma_remote() and "commaai" in get_origin(default="")):
+    return
+
+  env = "production" if is_tested_branch() else "master"
+  dongle_id = Params().get("DongleId", encoding='utf-8')
+
+  integrations = []
+  if project == SentryProject.SELFDRIVE:
+    integrations.append(ThreadingIntegration(propagate_hub=True))
+  else:
+    sentry_sdk.utils.MAX_STRING_LENGTH = 8192
+
+  sentry_sdk.init(project.value,
+                  default_integrations=False,
+                  release=get_version(),
+                  integrations=integrations,
+                  traces_sample_rate=1.0,
+                  environment=env)
+
+  sentry_sdk.set_user({"id": dongle_id})
+  sentry_sdk.set_tag("dirty", is_dirty())
+  sentry_sdk.set_tag("origin", get_origin())
+  sentry_sdk.set_tag("branch", get_branch())
+  sentry_sdk.set_tag("commit", get_commit())
+  sentry_sdk.set_tag("device", HARDWARE.get_device_type())
+
+  if project == SentryProject.SELFDRIVE:
+    sentry_sdk.Hub.current.start_session()

selfdrive/tombstoned.py

@@ -9,14 +9,12 @@ import time
 import glob
 from typing import NoReturn
 
-import sentry_sdk
-
-from common.params import Params
 from common.file_helpers import mkdirs_exists_ok
-from selfdrive.hardware import TICI, HARDWARE
+from selfdrive.hardware import TICI
 from selfdrive.loggerd.config import ROOT
+import selfdrive.sentry as sentry
 from selfdrive.swaglog import cloudlog
-from selfdrive.version import get_branch, get_commit, is_dirty, get_origin, get_version
+from selfdrive.version import get_commit
 
 MAX_SIZE = 100000 * 10  # mal size is 40-100k, allow up to 1M
 if TICI:
@@ -32,16 +30,6 @@ def safe_fn(s):
   return "".join(c for c in s if c.isalnum() or c in extra).rstrip()
 
 
-def sentry_report(fn, message, contents):
-  cloudlog.error({'tombstone': message})
-
-  with sentry_sdk.configure_scope() as scope:
-      scope.set_extra("tombstone_fn", fn)
-      scope.set_extra("tombstone", contents)
-      sentry_sdk.capture_message(message=message)
-      sentry_sdk.flush()
-
-
 def clear_apport_folder():
   for f in glob.glob(APPORT_DIR + '*'):
     try:
@@ -104,7 +92,7 @@ def report_tombstone_android(fn):
   if fault_idx >= 0:
     message = message[:fault_idx]
 
-  sentry_report(fn, message, contents)
+  sentry.report_tombstone(fn, message, contents)
 
   # Copy crashlog to upload folder
   clean_path = executable.replace('./', '').replace('/', '_')
@@ -178,7 +166,7 @@ def report_tombstone_apport(fn):
 
   contents = stacktrace + "\n\n" + contents
   message = message + " - " + crash_function
-  sentry_report(fn, message, contents)
+  sentry.report_tombstone(fn, message, contents)
 
   # Copy crashlog to upload folder
   clean_path = path.replace('/', '_')
@@ -199,20 +187,11 @@ def report_tombstone_apport(fn):
 
 
 def main() -> NoReturn:
-  clear_apport_folder()  # Clear apport folder on start, otherwise duplicate crashes won't register
+  sentry.init(sentry.SentryProject.SELFDRIVE_NATIVE)
-  initial_tombstones = set(get_tombstones())
 
-  sentry_sdk.utils.MAX_STRING_LENGTH = 8192
+  # Clear apport folder on start, otherwise duplicate crashes won't register
-  sentry_sdk.init("https://a40f22e13cbc4261873333c125fc9d38@o33823.ingest.sentry.io/157615",
+  clear_apport_folder()
-                  default_integrations=False, release=get_version())
+  initial_tombstones = set(get_tombstones())
-
-  dongle_id = Params().get("DongleId", encoding='utf-8')
-  sentry_sdk.set_user({"id": dongle_id})
-  sentry_sdk.set_tag("dirty", is_dirty())
-  sentry_sdk.set_tag("origin", get_origin())
-  sentry_sdk.set_tag("branch", get_branch())
-  sentry_sdk.set_tag("commit", get_commit())
-  sentry_sdk.set_tag("device", HARDWARE.get_device_type())
 
   while True:
     now_tombstones = set(get_tombstones())

selfdrive/version.py

@@ -79,6 +79,8 @@ def is_prebuilt() -> bool:
 
 @cache
 def is_comma_remote() -> bool:
+  # note to fork maintainers, this is used for release metrics. please do not
+  # touch this to get rid of the orange startup alert. there's better ways to do that
   origin = get_origin()
   if origin is None:
     return False