diff --git a/release/files_common b/release/files_common index 23b80ffa74..8d682d383a 100644 --- a/release/files_common +++ b/release/files_common @@ -70,7 +70,7 @@ selfdrive/version.py selfdrive/__init__.py selfdrive/config.py -selfdrive/crash.py +selfdrive/sentry.py selfdrive/swaglog.py selfdrive/logmessaged.py selfdrive/tombstoned.py diff --git a/selfdrive/crash.py b/selfdrive/crash.py deleted file mode 100644 index e42b7532b0..0000000000 --- a/selfdrive/crash.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Install exception handler for process crash.""" -from selfdrive.swaglog import cloudlog -from selfdrive.version import get_version - -import sentry_sdk -from sentry_sdk.integrations.threading import ThreadingIntegration - -def capture_exception(*args, **kwargs) -> None: - cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1)) - - try: - sentry_sdk.capture_exception(*args, **kwargs) - sentry_sdk.flush() # https://github.com/getsentry/sentry-python/issues/291 - except Exception: - cloudlog.exception("sentry exception") - -def bind_user(**kwargs) -> None: - sentry_sdk.set_user(kwargs) - -def bind_extra(**kwargs) -> None: - for k, v in kwargs.items(): - sentry_sdk.set_tag(k, v) - -def init() -> None: - sentry_sdk.init("https://a8dc76b5bfb34908a601d67e2aa8bcf9@o33823.ingest.sentry.io/77924", - default_integrations=False, integrations=[ThreadingIntegration(propagate_hub=True)], - release=get_version()) diff --git a/selfdrive/manager/manager.py b/selfdrive/manager/manager.py index 15ede773ed..77fb9543d2 100755 --- a/selfdrive/manager/manager.py +++ b/selfdrive/manager/manager.py @@ -8,7 +8,7 @@ import traceback from typing import List, Tuple, Union import cereal.messaging as messaging -import selfdrive.crash as crash +import selfdrive.sentry as sentry from common.basedir import BASEDIR from common.params import Params, ParamKeyType from common.text_window import TextWindow @@ -20,7 +20,7 @@ from selfdrive.manager.process_config import managed_processes from selfdrive.athena.registration import register, UNREGISTERED_DONGLE_ID from selfdrive.swaglog import cloudlog, add_file_handler from selfdrive.version import is_dirty, get_commit, get_version, get_origin, get_short_branch, \ - terms_version, training_version, is_comma_remote + terms_version, training_version sys.path.append(os.path.join(BASEDIR, "pyextra")) @@ -90,15 +90,11 @@ def manager_init() -> None: if not is_dirty(): os.environ['CLEAN'] = '1' + # init logging + sentry.init(sentry.SentryProject.SELFDRIVE) cloudlog.bind_global(dongle_id=dongle_id, version=get_version(), dirty=is_dirty(), device=HARDWARE.get_device_type()) - if is_comma_remote() and not (os.getenv("NOLOG") or os.getenv("NOCRASH") or PC): - crash.init() - crash.bind_user(id=dongle_id) - crash.bind_extra(dirty=is_dirty(), origin=get_origin(), branch=get_short_branch(), commit=get_commit(), - device=HARDWARE.get_device_type()) - def manager_prepare() -> None: for p in managed_processes.values(): @@ -195,7 +191,7 @@ def main() -> None: manager_thread() except Exception: traceback.print_exc() - crash.capture_exception() + sentry.capture_exception() finally: manager_cleanup() diff --git a/selfdrive/manager/process.py b/selfdrive/manager/process.py index 42dbab05ea..5165d6da78 100644 --- a/selfdrive/manager/process.py +++ b/selfdrive/manager/process.py @@ -11,7 +11,7 @@ from multiprocessing import Process from setproctitle import setproctitle # pylint: disable=no-name-in-module import cereal.messaging as messaging -import selfdrive.crash as crash +import selfdrive.sentry as sentry from common.basedir import BASEDIR from common.params import Params from common.realtime import sec_since_boot @@ -44,7 +44,7 @@ def launcher(proc: str, name: str) -> None: except Exception: # can't install the crash handler because sys.excepthook doesn't play nice # with threads, so catch it here. - crash.capture_exception() + sentry.capture_exception() raise diff --git a/selfdrive/sentry.py b/selfdrive/sentry.py new file mode 100644 index 0000000000..85c97a9873 --- /dev/null +++ b/selfdrive/sentry.py @@ -0,0 +1,69 @@ +"""Install exception handler for process crash.""" +import sentry_sdk +from enum import Enum +from sentry_sdk.integrations.threading import ThreadingIntegration + +from common.params import Params +from selfdrive.hardware import HARDWARE +from selfdrive.swaglog import cloudlog +from selfdrive.version import get_branch, get_commit, get_origin, get_version, \ + is_comma_remote, is_dirty, is_tested_branch + + +class SentryProject(Enum): + # python project + SELFDRIVE = "https://a8dc76b5bfb34908a601d67e2aa8bcf9@o33823.ingest.sentry.io/77924" + # native project + SELFDRIVE_NATIVE = "https://a40f22e13cbc4261873333c125fc9d38@o33823.ingest.sentry.io/157615" + + +def report_tombstone(fn: str, message: str, contents: str) -> None: + cloudlog.error({'tombstone': message}) + + with sentry_sdk.configure_scope() as scope: + scope.set_extra("tombstone_fn", fn) + scope.set_extra("tombstone", contents) + sentry_sdk.capture_message(message=message) + sentry_sdk.flush() + + +def capture_exception(*args, **kwargs) -> None: + cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1)) + + try: + sentry_sdk.capture_exception(*args, **kwargs) + sentry_sdk.flush() # https://github.com/getsentry/sentry-python/issues/291 + except Exception: + cloudlog.exception("sentry exception") + + +def init(project: SentryProject) -> None: + # forks like to mess with this, so double check + if not (is_comma_remote() and "commaai" in get_origin(default="")): + return + + env = "production" if is_tested_branch() else "master" + dongle_id = Params().get("DongleId", encoding='utf-8') + + integrations = [] + if project == SentryProject.SELFDRIVE: + integrations.append(ThreadingIntegration(propagate_hub=True)) + else: + sentry_sdk.utils.MAX_STRING_LENGTH = 8192 + + sentry_sdk.init(project.value, + default_integrations=False, + release=get_version(), + integrations=integrations, + traces_sample_rate=1.0, + environment=env) + + sentry_sdk.set_user({"id": dongle_id}) + sentry_sdk.set_tag("dirty", is_dirty()) + sentry_sdk.set_tag("origin", get_origin()) + sentry_sdk.set_tag("branch", get_branch()) + sentry_sdk.set_tag("commit", get_commit()) + sentry_sdk.set_tag("device", HARDWARE.get_device_type()) + + if project == SentryProject.SELFDRIVE: + sentry_sdk.Hub.current.start_session() diff --git a/selfdrive/tombstoned.py b/selfdrive/tombstoned.py index df96c222c0..459ab2f0f9 100755 --- a/selfdrive/tombstoned.py +++ b/selfdrive/tombstoned.py @@ -9,14 +9,12 @@ import time import glob from typing import NoReturn -import sentry_sdk - -from common.params import Params from common.file_helpers import mkdirs_exists_ok -from selfdrive.hardware import TICI, HARDWARE +from selfdrive.hardware import TICI from selfdrive.loggerd.config import ROOT +import selfdrive.sentry as sentry from selfdrive.swaglog import cloudlog -from selfdrive.version import get_branch, get_commit, is_dirty, get_origin, get_version +from selfdrive.version import get_commit MAX_SIZE = 100000 * 10 # mal size is 40-100k, allow up to 1M if TICI: @@ -32,16 +30,6 @@ def safe_fn(s): return "".join(c for c in s if c.isalnum() or c in extra).rstrip() -def sentry_report(fn, message, contents): - cloudlog.error({'tombstone': message}) - - with sentry_sdk.configure_scope() as scope: - scope.set_extra("tombstone_fn", fn) - scope.set_extra("tombstone", contents) - sentry_sdk.capture_message(message=message) - sentry_sdk.flush() - - def clear_apport_folder(): for f in glob.glob(APPORT_DIR + '*'): try: @@ -104,7 +92,7 @@ def report_tombstone_android(fn): if fault_idx >= 0: message = message[:fault_idx] - sentry_report(fn, message, contents) + sentry.report_tombstone(fn, message, contents) # Copy crashlog to upload folder clean_path = executable.replace('./', '').replace('/', '_') @@ -178,7 +166,7 @@ def report_tombstone_apport(fn): contents = stacktrace + "\n\n" + contents message = message + " - " + crash_function - sentry_report(fn, message, contents) + sentry.report_tombstone(fn, message, contents) # Copy crashlog to upload folder clean_path = path.replace('/', '_') @@ -199,20 +187,11 @@ def report_tombstone_apport(fn): def main() -> NoReturn: - clear_apport_folder() # Clear apport folder on start, otherwise duplicate crashes won't register - initial_tombstones = set(get_tombstones()) + sentry.init(sentry.SentryProject.SELFDRIVE_NATIVE) - sentry_sdk.utils.MAX_STRING_LENGTH = 8192 - sentry_sdk.init("https://a40f22e13cbc4261873333c125fc9d38@o33823.ingest.sentry.io/157615", - default_integrations=False, release=get_version()) - - dongle_id = Params().get("DongleId", encoding='utf-8') - sentry_sdk.set_user({"id": dongle_id}) - sentry_sdk.set_tag("dirty", is_dirty()) - sentry_sdk.set_tag("origin", get_origin()) - sentry_sdk.set_tag("branch", get_branch()) - sentry_sdk.set_tag("commit", get_commit()) - sentry_sdk.set_tag("device", HARDWARE.get_device_type()) + # Clear apport folder on start, otherwise duplicate crashes won't register + clear_apport_folder() + initial_tombstones = set(get_tombstones()) while True: now_tombstones = set(get_tombstones()) diff --git a/selfdrive/version.py b/selfdrive/version.py index 0c1a44f236..d94a098e8b 100644 --- a/selfdrive/version.py +++ b/selfdrive/version.py @@ -79,6 +79,8 @@ def is_prebuilt() -> bool: @cache def is_comma_remote() -> bool: + # note to fork maintainers, this is used for release metrics. please do not + # touch this to get rid of the orange startup alert. there's better ways to do that origin = get_origin() if origin is None: return False