From effee900c4e9f3950b88a43b08da40bdbc47aed0 Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Thu, 21 Mar 2024 13:15:29 -0400 Subject: [PATCH] use build_metadata everywhere we used to use get_version, get_commit, etc (#31941) * use build_metadata * fix normailzed * also normalized * and here * fix diff * and that one * cleanup --- selfdrive/athena/athenad.py | 12 ++-- selfdrive/athena/manage_athenad.py | 15 ++--- selfdrive/car/car_helpers.py | 5 +- selfdrive/manager/build.py | 5 +- selfdrive/manager/manager.py | 41 +++++++------- selfdrive/sentry.py | 18 +++--- selfdrive/statsd.py | 12 ++-- selfdrive/tombstoned.py | 6 +- selfdrive/updated/updated.py | 5 +- system/version.py | 88 ++++++++++++++++++------------ 10 files changed, 118 insertions(+), 89 deletions(-) diff --git a/selfdrive/athena/athenad.py b/selfdrive/athena/athenad.py index d228af572c..989e284e74 100755 --- a/selfdrive/athena/athenad.py +++ b/selfdrive/athena/athenad.py @@ -32,13 +32,12 @@ from cereal import log from cereal.services import SERVICE_LIST from openpilot.common.api import Api from openpilot.common.file_helpers import CallbackReader -from openpilot.common.git import get_commit, get_normalized_origin, get_short_branch from openpilot.common.params import Params from openpilot.common.realtime import set_core_affinity from openpilot.system.hardware import HARDWARE, PC from openpilot.system.loggerd.xattr_cache import getxattr, setxattr from openpilot.common.swaglog import cloudlog -from openpilot.system.version import get_version +from openpilot.system.version import get_build_metadata from openpilot.system.hardware.hw import Paths @@ -320,11 +319,12 @@ def getMessage(service: str, timeout: int = 1000) -> dict: @dispatcher.add_method def getVersion() -> dict[str, str]: + build_metadata = get_build_metadata() return { - "version": get_version(), - "remote": get_normalized_origin(), - "branch": get_short_branch(), - "commit": get_commit(), + "version": build_metadata.openpilot.version, + "remote": build_metadata.openpilot.git_normalized_origin, + "branch": build_metadata.channel, + "commit": build_metadata.openpilot.git_commit, } diff --git a/selfdrive/athena/manage_athenad.py b/selfdrive/athena/manage_athenad.py index 3065bed5c7..2ec92cc3e0 100755 --- a/selfdrive/athena/manage_athenad.py +++ b/selfdrive/athena/manage_athenad.py @@ -7,8 +7,7 @@ from openpilot.common.params import Params from openpilot.selfdrive.manager.process import launcher from openpilot.common.swaglog import cloudlog from openpilot.system.hardware import HARDWARE -from openpilot.common.git import get_commit, get_normalized_origin, get_short_branch -from openpilot.system.version import get_version, is_dirty +from openpilot.system.version import get_build_metadata ATHENA_MGR_PID_PARAM = "AthenadPid" @@ -16,12 +15,14 @@ ATHENA_MGR_PID_PARAM = "AthenadPid" def main(): params = Params() dongle_id = params.get("DongleId").decode('utf-8') + build_metadata = get_build_metadata() + cloudlog.bind_global(dongle_id=dongle_id, - version=get_version(), - origin=get_normalized_origin(), - branch=get_short_branch(), - commit=get_commit(), - dirty=is_dirty(), + version=build_metadata.openpilot.version, + origin=build_metadata.openpilot.git_normalized_origin, + branch=build_metadata.channel, + commit=build_metadata.openpilot.git_commit, + dirty=build_metadata.openpilot.is_dirty, device=HARDWARE.get_device_type()) try: diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py index fd8ecc5020..0e2443d330 100644 --- a/selfdrive/car/car_helpers.py +++ b/selfdrive/car/car_helpers.py @@ -4,7 +4,6 @@ from collections.abc import Callable from cereal import car from openpilot.common.params import Params -from openpilot.system.version import is_comma_remote, is_tested_branch from openpilot.selfdrive.car.interfaces import get_interface_attr from openpilot.selfdrive.car.fingerprints import eliminate_incompatible_cars, all_legacy_fingerprint_cars from openpilot.selfdrive.car.vin import get_vin, is_valid_vin, VIN_UNKNOWN @@ -13,6 +12,7 @@ from openpilot.selfdrive.car.mock.values import CAR as MOCK from openpilot.common.swaglog import cloudlog import cereal.messaging as messaging from openpilot.selfdrive.car import gen_empty_fingerprint +from openpilot.system.version import get_build_metadata FRAME_FINGERPRINT = 100 # 1s @@ -20,7 +20,8 @@ EventName = car.CarEvent.EventName def get_startup_event(car_recognized, controller_available, fw_seen): - if is_comma_remote() and is_tested_branch(): + build_metadata = get_build_metadata() + if build_metadata.openpilot.comma_remote and build_metadata.tested_channel: event = EventName.startup else: event = EventName.startupMaster diff --git a/selfdrive/manager/build.py b/selfdrive/manager/build.py index 067e1b5a1e..859d0920c3 100755 --- a/selfdrive/manager/build.py +++ b/selfdrive/manager/build.py @@ -9,7 +9,7 @@ from openpilot.common.spinner import Spinner from openpilot.common.text_window import TextWindow from openpilot.system.hardware import AGNOS from openpilot.common.swaglog import cloudlog, add_file_handler -from openpilot.system.version import is_dirty +from openpilot.system.version import get_build_metadata MAX_CACHE_SIZE = 4e9 if "CI" in os.environ else 2e9 CACHE_DIR = Path("/data/scons_cache" if AGNOS else "/tmp/scons_cache") @@ -86,4 +86,5 @@ def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None: if __name__ == "__main__": spinner = Spinner() spinner.update_progress(0, 100) - build(spinner, is_dirty(), minimal = AGNOS) + build_metadata = get_build_metadata() + build(spinner, build_metadata.openpilot.is_dirty, minimal = AGNOS) diff --git a/selfdrive/manager/manager.py b/selfdrive/manager/manager.py index f30b81861a..512320a8db 100755 --- a/selfdrive/manager/manager.py +++ b/selfdrive/manager/manager.py @@ -16,21 +16,20 @@ from openpilot.selfdrive.manager.process import ensure_running from openpilot.selfdrive.manager.process_config import managed_processes from openpilot.selfdrive.athena.registration import register, UNREGISTERED_DONGLE_ID from openpilot.common.swaglog import cloudlog, add_file_handler -from openpilot.common.git import get_commit, get_origin, get_short_branch, get_commit_date -from openpilot.system.version import is_dirty, get_version, \ - get_normalized_origin, terms_version, training_version, \ - is_tested_branch, is_release_branch +from openpilot.system.version import get_build_metadata, terms_version, training_version def manager_init() -> None: save_bootlog() + build_metadata = get_build_metadata() + params = Params() params.clear_all(ParamKeyType.CLEAR_ON_MANAGER_START) params.clear_all(ParamKeyType.CLEAR_ON_ONROAD_TRANSITION) params.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION) - if is_release_branch(): + if build_metadata.release_channel: params.clear_all(ParamKeyType.DEVELOPMENT_ONLY) default_params: list[tuple[str, str | bytes]] = [ @@ -62,15 +61,15 @@ def manager_init() -> None: print("WARNING: failed to make /dev/shm") # set version params - params.put("Version", get_version()) + params.put("Version", build_metadata.openpilot.version) params.put("TermsVersion", terms_version) params.put("TrainingVersion", training_version) - params.put("GitCommit", get_commit()) - params.put("GitCommitDate", get_commit_date()) - params.put("GitBranch", get_short_branch()) - params.put("GitRemote", get_origin()) - params.put_bool("IsTestedBranch", is_tested_branch()) - params.put_bool("IsReleaseBranch", is_release_branch()) + params.put("GitCommit", build_metadata.openpilot.git_commit) + params.put("GitCommitDate", build_metadata.openpilot.git_commit_date) + params.put("GitBranch", build_metadata.channel) + params.put("GitRemote", build_metadata.openpilot.git_origin) + params.put_bool("IsTestedBranch", build_metadata.tested_channel) + params.put_bool("IsReleaseBranch", build_metadata.release_channel) # set dongle id reg_res = register(show_spinner=True) @@ -80,21 +79,21 @@ def manager_init() -> None: serial = params.get("HardwareSerial") raise Exception(f"Registration failed for device {serial}") os.environ['DONGLE_ID'] = dongle_id # Needed for swaglog - os.environ['GIT_ORIGIN'] = get_normalized_origin() # Needed for swaglog - os.environ['GIT_BRANCH'] = get_short_branch() # Needed for swaglog - os.environ['GIT_COMMIT'] = get_commit() # Needed for swaglog + os.environ['GIT_ORIGIN'] = build_metadata.openpilot.git_normalized_origin # Needed for swaglog + os.environ['GIT_BRANCH'] = build_metadata.channel # Needed for swaglog + os.environ['GIT_COMMIT'] = build_metadata.openpilot.git_commit # Needed for swaglog - if not is_dirty(): + if not build_metadata.openpilot.is_dirty: os.environ['CLEAN'] = '1' # init logging sentry.init(sentry.SentryProject.SELFDRIVE) cloudlog.bind_global(dongle_id=dongle_id, - version=get_version(), - origin=get_normalized_origin(), - branch=get_short_branch(), - commit=get_commit(), - dirty=is_dirty(), + version=build_metadata.openpilot.version, + origin=build_metadata.openpilot.git_normalized_origin, + branch=build_metadata.channel, + commit=build_metadata.openpilot.git_commit, + dirty=build_metadata.openpilot.is_dirty, device=HARDWARE.get_device_type()) # preimport all processes diff --git a/selfdrive/sentry.py b/selfdrive/sentry.py index 889178610f..204d9cab09 100644 --- a/selfdrive/sentry.py +++ b/selfdrive/sentry.py @@ -6,9 +6,8 @@ from sentry_sdk.integrations.threading import ThreadingIntegration from openpilot.common.params import Params from openpilot.selfdrive.athena.registration import is_registered_device from openpilot.system.hardware import HARDWARE, PC -from openpilot.common.git import get_commit, get_branch, get_origin from openpilot.common.swaglog import cloudlog -from openpilot.system.version import get_version, is_comma_remote, is_dirty, is_tested_branch +from openpilot.system.version import get_build_metadata, get_version class SentryProject(Enum): @@ -43,12 +42,13 @@ def set_tag(key: str, value: str) -> None: def init(project: SentryProject) -> bool: + build_metadata = get_build_metadata() # forks like to mess with this, so double check - comma_remote = is_comma_remote() and "commaai" in get_origin() + comma_remote = build_metadata.openpilot.comma_remote and "commaai" in build_metadata.openpilot.git_origin if not comma_remote or not is_registered_device() or PC: return False - env = "release" if is_tested_branch() else "master" + env = "release" if build_metadata.tested_channel else "master" dongle_id = Params().get("DongleId", encoding='utf-8') integrations = [] @@ -63,11 +63,13 @@ def init(project: SentryProject) -> bool: max_value_length=8192, environment=env) + build_metadata = get_build_metadata() + sentry_sdk.set_user({"id": dongle_id}) - sentry_sdk.set_tag("dirty", is_dirty()) - sentry_sdk.set_tag("origin", get_origin()) - sentry_sdk.set_tag("branch", get_branch()) - sentry_sdk.set_tag("commit", get_commit()) + sentry_sdk.set_tag("dirty", build_metadata.openpilot.is_dirty) + sentry_sdk.set_tag("origin", build_metadata.openpilot.git_origin) + sentry_sdk.set_tag("branch", build_metadata.channel) + sentry_sdk.set_tag("commit", build_metadata.openpilot.git_commit) sentry_sdk.set_tag("device", HARDWARE.get_device_type()) if project == SentryProject.SELFDRIVE: diff --git a/selfdrive/statsd.py b/selfdrive/statsd.py index 299aa295d7..2f9149b096 100755 --- a/selfdrive/statsd.py +++ b/selfdrive/statsd.py @@ -13,7 +13,7 @@ from openpilot.system.hardware.hw import Paths from openpilot.common.swaglog import cloudlog from openpilot.system.hardware import HARDWARE from openpilot.common.file_helpers import atomic_write_in_dir -from openpilot.system.version import get_normalized_origin, get_short_branch, get_short_version, is_dirty +from openpilot.system.version import get_build_metadata from openpilot.system.loggerd.config import STATS_DIR_FILE_LIMIT, STATS_SOCKET, STATS_FLUSH_TIME_S @@ -86,13 +86,15 @@ def main() -> NoReturn: # initialize stats directory Path(STATS_DIR).mkdir(parents=True, exist_ok=True) + build_metadata = get_build_metadata() + # initialize tags tags = { 'started': False, - 'version': get_short_version(), - 'branch': get_short_branch(), - 'dirty': is_dirty(), - 'origin': get_normalized_origin(), + 'version': build_metadata.openpilot.version, + 'branch': build_metadata.channel, + 'dirty': build_metadata.openpilot.is_dirty, + 'origin': build_metadata.openpilot.git_normalized_origin, 'deviceType': HARDWARE.get_device_type(), } diff --git a/selfdrive/tombstoned.py b/selfdrive/tombstoned.py index f1b8c88083..2c99c7eafe 100755 --- a/selfdrive/tombstoned.py +++ b/selfdrive/tombstoned.py @@ -11,8 +11,8 @@ from typing import NoReturn import openpilot.selfdrive.sentry as sentry from openpilot.system.hardware.hw import Paths -from openpilot.common.git import get_commit from openpilot.common.swaglog import cloudlog +from openpilot.system.version import get_build_metadata MAX_SIZE = 1_000_000 * 100 # allow up to 100M MAX_TOMBSTONE_FN_LEN = 62 # 85 - 23 ("/crash/") @@ -124,7 +124,9 @@ def report_tombstone_apport(fn): clean_path = path.replace('/', '_') date = datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S") - new_fn = f"{date}_{(get_commit() or 'nocommit')[:8]}_{safe_fn(clean_path)}"[:MAX_TOMBSTONE_FN_LEN] + build_metadata = get_build_metadata() + + new_fn = f"{date}_{(build_metadata.openpilot.git_commit or 'nocommit')[:8]}_{safe_fn(clean_path)}"[:MAX_TOMBSTONE_FN_LEN] crashlog_dir = os.path.join(Paths.log_root(), "crash") os.makedirs(crashlog_dir, exist_ok=True) diff --git a/selfdrive/updated/updated.py b/selfdrive/updated/updated.py index b6b395f254..3a710ba02f 100755 --- a/selfdrive/updated/updated.py +++ b/selfdrive/updated/updated.py @@ -19,7 +19,7 @@ from openpilot.common.time import system_time_valid from openpilot.system.hardware import AGNOS, HARDWARE from openpilot.common.swaglog import cloudlog from openpilot.selfdrive.controls.lib.alertmanager import set_offroad_alert -from openpilot.system.version import is_tested_branch +from openpilot.system.version import get_build_metadata LOCK_FILE = os.getenv("UPDATER_LOCK_FILE", "/tmp/safe_staging_overlay.lock") STAGING_ROOT = os.getenv("UPDATER_STAGING_ROOT", "/data/safe_staging") @@ -325,8 +325,9 @@ class Updater: now = datetime.datetime.utcnow() dt = now - last_update + build_metadata = get_build_metadata() if failed_count > 15 and exception is not None and self.has_internet: - if is_tested_branch(): + if build_metadata.tested_channel: extra_text = "Ensure the software is correctly installed. Uninstall and re-install if this error persists." else: extra_text = exception diff --git a/system/version.py b/system/version.py index 7b3ea940f4..f370beb4ae 100755 --- a/system/version.py +++ b/system/version.py @@ -9,7 +9,7 @@ import subprocess from openpilot.common.basedir import BASEDIR from openpilot.common.swaglog import cloudlog from openpilot.common.utils import cache -from openpilot.common.git import get_commit, get_origin, get_branch, get_short_branch, get_normalized_origin, get_commit_date +from openpilot.common.git import get_commit, get_origin, get_branch, get_short_branch, get_commit_date RELEASE_BRANCHES = ['release3-staging', 'release3', 'nightly'] @@ -32,33 +32,15 @@ def get_release_notes(path: str = BASEDIR) -> str: return f.read().split('\n\n', 1)[0] -@cache -def get_short_version() -> str: - return get_version().split('-')[0] - @cache def is_prebuilt(path: str = BASEDIR) -> bool: return os.path.exists(os.path.join(path, 'prebuilt')) -@cache -def is_comma_remote() -> bool: - # note to fork maintainers, this is used for release metrics. please do not - # touch this to get rid of the orange startup alert. there's better ways to do that - return get_normalized_origin() == "github.com/commaai/openpilot" - -@cache -def is_tested_branch() -> bool: - return get_short_branch() in TESTED_BRANCHES - -@cache -def is_release_branch() -> bool: - return get_short_branch() in RELEASE_BRANCHES - @cache def is_dirty(cwd: str = BASEDIR) -> bool: - origin = get_origin(cwd) - branch = get_branch(cwd) + origin = get_origin() + branch = get_branch() if not origin or not branch: return True @@ -85,6 +67,27 @@ class OpenpilotMetadata: version: str release_notes: str git_commit: str + git_origin: str + git_commit_date: str + is_dirty: bool # whether there are local changes + + @property + def short_version(self) -> str: + return self.version.split('-')[0] + + @property + def comma_remote(self) -> bool: + # note to fork maintainers, this is used for release metrics. please do not + # touch this to get rid of the orange startup alert. there's better ways to do that + return self.git_normalized_origin == "github.com/commaai/openpilot" + + @property + def git_normalized_origin(self) -> str: + return self.git_origin \ + .replace("git@", "", 1) \ + .replace(".git", "", 1) \ + .replace("https://", "", 1) \ + .replace(":", "/", 1) @dataclass(frozen=True) @@ -92,9 +95,17 @@ class BuildMetadata: channel: str openpilot: OpenpilotMetadata + @property + def tested_channel(self) -> bool: + return self.channel in TESTED_BRANCHES + + @property + def release_channel(self) -> bool: + return self.channel in RELEASE_BRANCHES + -def get_build_metadata(path: str = BASEDIR) -> BuildMetadata | None: +def get_build_metadata(path: str = BASEDIR) -> BuildMetadata: build_metadata_path = pathlib.Path(path) / BUILD_METADATA_FILENAME if build_metadata_path.exists(): @@ -105,14 +116,31 @@ def get_build_metadata(path: str = BASEDIR) -> BuildMetadata | None: version = openpilot_metadata.get("version", "unknown") release_notes = openpilot_metadata.get("release_notes", "unknown") git_commit = openpilot_metadata.get("git_commit", "unknown") - return BuildMetadata(channel, OpenpilotMetadata(version, release_notes, git_commit)) + git_origin = openpilot_metadata.get("git_origin", "unknown") + git_commit_date = openpilot_metadata.get("git_commit_date", "unknown") + return BuildMetadata(channel, + OpenpilotMetadata( + version=version, + release_notes=release_notes, + git_commit=git_commit, + git_origin=git_origin, + git_commit_date=git_commit_date, + is_dirty=False)) git_folder = pathlib.Path(path) / ".git" if git_folder.exists(): - return BuildMetadata(get_short_branch(path), OpenpilotMetadata(get_version(path), get_release_notes(path), get_commit(path))) + return BuildMetadata(get_short_branch(path), + OpenpilotMetadata( + version=get_version(path), + release_notes=get_release_notes(path), + git_commit=get_commit(path), + git_origin=get_origin(path), + git_commit_date=get_commit_date(path), + is_dirty=is_dirty(path))) - return None + cloudlog.exception("unable to get build metadata") + raise Exception("invalid build metadata") if __name__ == "__main__": @@ -122,12 +150,4 @@ if __name__ == "__main__": params.put("TermsVersion", terms_version) params.put("TrainingVersion", training_version) - print(f"Dirty: {is_dirty()}") - print(f"Version: {get_version()}") - print(f"Short version: {get_short_version()}") - print(f"Origin: {get_origin()}") - print(f"Normalized origin: {get_normalized_origin()}") - print(f"Branch: {get_branch()}") - print(f"Short branch: {get_short_branch()}") - print(f"Prebuilt: {is_prebuilt()}") - print(f"Commit date: {get_commit_date()}") + print(get_build_metadata())