From a0b589eda9c749c697fa4db80b6129b2545e8fee Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Wed, 20 Mar 2024 17:35:19 -0500 Subject: [PATCH 1/7] Honda: allow fingerprinting without comma power for more platforms (#31935) * 4 more platforms * only the first is missing srs?! * vsa didn't respond on these 8 routes * acura is good! * do CRV Hybrid * CRV is already done * new line * Revert "new line" This reverts commit 411c92c77b695d3df716f84b6f302fa0f791d555. --- selfdrive/car/honda/values.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/selfdrive/car/honda/values.py b/selfdrive/car/honda/values.py index 32846f87a2..a050cf8e2a 100644 --- a/selfdrive/car/honda/values.py +++ b/selfdrive/car/honda/values.py @@ -311,14 +311,14 @@ FW_QUERY_CONFIG = FwQueryConfig( Ecu.transmission: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_CIVIC, CAR.HONDA_CIVIC_BOSCH, CAR.HONDA_CRV_5G], Ecu.srs: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_E], Ecu.eps: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_E], - Ecu.vsa: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_CIVIC, CAR.HONDA_CIVIC_BOSCH, CAR.HONDA_CRV_5G, CAR.HONDA_E], - Ecu.combinationMeter: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_CIVIC, CAR.HONDA_CIVIC_BOSCH, - CAR.HONDA_HRV, CAR.HONDA_CRV_5G, CAR.HONDA_E, CAR.HONDA_ODYSSEY_CHN], - Ecu.gateway: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_CIVIC, CAR.HONDA_CIVIC_BOSCH, CAR.HONDA_FREED, - CAR.HONDA_HRV, CAR.HONDA_CRV_5G, CAR.HONDA_E, CAR.HONDA_ODYSSEY_CHN], + Ecu.vsa: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_CIVIC, CAR.HONDA_CIVIC_BOSCH, CAR.HONDA_CRV_5G, CAR.HONDA_CRV_HYBRID, CAR.HONDA_E], + Ecu.combinationMeter: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_CIVIC, CAR.HONDA_CIVIC_BOSCH, CAR.HONDA_FIT, + CAR.HONDA_HRV, CAR.HONDA_CRV_5G, CAR.HONDA_CRV_HYBRID, CAR.HONDA_E, CAR.HONDA_ODYSSEY_CHN], + Ecu.gateway: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_CIVIC, CAR.HONDA_CIVIC_BOSCH, CAR.HONDA_FIT, CAR.HONDA_FREED, + CAR.HONDA_HRV, CAR.HONDA_CRV_5G, CAR.HONDA_CRV_HYBRID, CAR.HONDA_E, CAR.HONDA_ODYSSEY_CHN], Ecu.electricBrakeBooster: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_CIVIC_BOSCH, CAR.HONDA_CRV_5G], # existence correlates with transmission type for Accord ICE - Ecu.shiftByWire: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_E], + Ecu.shiftByWire: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_CRV_HYBRID, CAR.HONDA_E], # existence correlates with trim level Ecu.hud: [CAR.HONDA_ACCORD], }, From 1a03da9df336c136c7bf0e93cb2d2f3731f89555 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Wed, 20 Mar 2024 18:19:18 -0500 Subject: [PATCH 2/7] Honda Ridgeline and Insight: allow fingerprinting without comma power (#31938) * Ridgeline and Insight * ridgeline * Insight --- selfdrive/car/honda/values.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/selfdrive/car/honda/values.py b/selfdrive/car/honda/values.py index a050cf8e2a..7a8c4314ac 100644 --- a/selfdrive/car/honda/values.py +++ b/selfdrive/car/honda/values.py @@ -311,14 +311,15 @@ FW_QUERY_CONFIG = FwQueryConfig( Ecu.transmission: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_CIVIC, CAR.HONDA_CIVIC_BOSCH, CAR.HONDA_CRV_5G], Ecu.srs: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_E], Ecu.eps: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_E], - Ecu.vsa: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_CIVIC, CAR.HONDA_CIVIC_BOSCH, CAR.HONDA_CRV_5G, CAR.HONDA_CRV_HYBRID, CAR.HONDA_E], + Ecu.vsa: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_CIVIC, CAR.HONDA_CIVIC_BOSCH, CAR.HONDA_CRV_5G, CAR.HONDA_CRV_HYBRID, + CAR.HONDA_E, CAR.HONDA_INSIGHT], Ecu.combinationMeter: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_CIVIC, CAR.HONDA_CIVIC_BOSCH, CAR.HONDA_FIT, - CAR.HONDA_HRV, CAR.HONDA_CRV_5G, CAR.HONDA_CRV_HYBRID, CAR.HONDA_E, CAR.HONDA_ODYSSEY_CHN], + CAR.HONDA_HRV, CAR.HONDA_CRV_5G, CAR.HONDA_CRV_HYBRID, CAR.HONDA_E, CAR.HONDA_INSIGHT, CAR.HONDA_ODYSSEY_CHN], Ecu.gateway: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_CIVIC, CAR.HONDA_CIVIC_BOSCH, CAR.HONDA_FIT, CAR.HONDA_FREED, - CAR.HONDA_HRV, CAR.HONDA_CRV_5G, CAR.HONDA_CRV_HYBRID, CAR.HONDA_E, CAR.HONDA_ODYSSEY_CHN], + CAR.HONDA_HRV, CAR.HONDA_RIDGELINE, CAR.HONDA_CRV_5G, CAR.HONDA_CRV_HYBRID, CAR.HONDA_E, CAR.HONDA_INSIGHT, CAR.HONDA_ODYSSEY_CHN], Ecu.electricBrakeBooster: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_CIVIC_BOSCH, CAR.HONDA_CRV_5G], # existence correlates with transmission type for Accord ICE - Ecu.shiftByWire: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_CRV_HYBRID, CAR.HONDA_E], + Ecu.shiftByWire: [CAR.ACURA_RDX_3G, CAR.HONDA_ACCORD, CAR.HONDA_CRV_HYBRID, CAR.HONDA_E, CAR.HONDA_INSIGHT], # existence correlates with trim level Ecu.hud: [CAR.HONDA_ACCORD], }, From 38d03b997958effea805a6c3860d2c5edaf2ae65 Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Wed, 20 Mar 2024 19:43:58 -0400 Subject: [PATCH 3/7] add get_build_metadata function (#31923) * version * Get build metadata * two lines * channel * cwd * default to unknown * dataclass --- common/git.py | 16 ++++++++-------- common/run.py | 8 ++++---- system/version.py | 42 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/common/git.py b/common/git.py index e15a5051d2..1ab8b87099 100644 --- a/common/git.py +++ b/common/git.py @@ -4,23 +4,23 @@ from openpilot.common.run import run_cmd, run_cmd_default @cache -def get_commit(branch: str = "HEAD") -> str: - return run_cmd_default(["git", "rev-parse", branch]) +def get_commit(cwd: str = None, branch: str = "HEAD") -> str: + return run_cmd_default(["git", "rev-parse", branch], cwd=cwd) @cache -def get_commit_date(commit: str = "HEAD") -> str: - return run_cmd_default(["git", "show", "--no-patch", "--format='%ct %ci'", commit]) +def get_commit_date(cwd: str = None, commit: str = "HEAD") -> str: + return run_cmd_default(["git", "show", "--no-patch", "--format='%ct %ci'", commit], cwd=cwd) @cache -def get_short_branch() -> str: - return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"]) +def get_short_branch(cwd: str = None) -> str: + return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=cwd) @cache -def get_branch() -> str: - return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"]) +def get_branch(cwd: str = None) -> str: + return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], cwd=cwd) @cache diff --git a/common/run.py b/common/run.py index 25abe98c41..c1a36bf041 100644 --- a/common/run.py +++ b/common/run.py @@ -1,13 +1,13 @@ import subprocess -def run_cmd(cmd: list[str]) -> str: - return subprocess.check_output(cmd, encoding='utf8').strip() +def run_cmd(cmd: list[str], cwd=None) -> str: + return subprocess.check_output(cmd, encoding='utf8', cwd=cwd).strip() -def run_cmd_default(cmd: list[str], default: str = "") -> str: +def run_cmd_default(cmd: list[str], default: str = "", cwd=None) -> str: try: - return run_cmd(cmd) + return run_cmd(cmd, cwd=cwd) except subprocess.CalledProcessError: return default diff --git a/system/version.py b/system/version.py index 7ae8313089..8866026152 100755 --- a/system/version.py +++ b/system/version.py @@ -1,17 +1,22 @@ #!/usr/bin/env python3 +from dataclasses import dataclass +import json import os +import pathlib import subprocess from openpilot.common.basedir import BASEDIR from openpilot.common.swaglog import cloudlog from openpilot.common.utils import cache -from openpilot.common.git import get_origin, get_branch, get_short_branch, get_normalized_origin, get_commit_date +from openpilot.common.git import get_commit, get_origin, get_branch, get_short_branch, get_normalized_origin, get_commit_date RELEASE_BRANCHES = ['release3-staging', 'release3', 'nightly'] TESTED_BRANCHES = RELEASE_BRANCHES + ['devel', 'devel-staging'] +BUILD_METADATA_FILENAME = "build.json" + training_version: bytes = b"0.2.0" terms_version: bytes = b"2" @@ -75,6 +80,41 @@ def is_dirty() -> bool: return dirty +@dataclass(frozen=True) +class OpenpilotMetadata: + version: str + release_notes: str + git_commit: str + + +@dataclass(frozen=True) +class BuildMetadata: + channel: str + openpilot: OpenpilotMetadata + + + +def get_build_metadata(path: str = BASEDIR) -> BuildMetadata | None: + build_metadata_path = pathlib.Path(path) / BUILD_METADATA_FILENAME + + if build_metadata_path.exists(): + build_metadata = json.loads(build_metadata_path.read_text()) + openpilot_metadata = build_metadata.get("openpilot", {}) + + channel = build_metadata.get("channel", "unknown") + version = openpilot_metadata.get("version", "unknown") + release_notes = openpilot_metadata.get("release_notes", "unknown") + git_commit = openpilot_metadata.get("git_commit", "unknown") + return BuildMetadata(channel, OpenpilotMetadata(version, release_notes, git_commit)) + + git_folder = pathlib.Path(path) / ".git" + + if git_folder.exists(): + return BuildMetadata(get_short_branch(path), OpenpilotMetadata(get_version(path), get_release_notes(path), get_commit(path))) + + return None + + if __name__ == "__main__": from openpilot.common.params import Params From 75b9b7b577e13039fb1faa22fb3f91785727fb81 Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Thu, 21 Mar 2024 00:07:54 +0000 Subject: [PATCH 4/7] use build_metadata --- common/git.py | 14 +++--- selfdrive/athena/athenad.py | 12 ++--- selfdrive/athena/manage_athenad.py | 15 +++--- selfdrive/car/car_helpers.py | 5 +- selfdrive/manager/build.py | 5 +- selfdrive/manager/manager.py | 41 ++++++++--------- selfdrive/sentry.py | 18 ++++---- selfdrive/statsd.py | 12 +++-- selfdrive/tombstoned.py | 6 ++- selfdrive/updated/updated.py | 5 +- system/version.py | 74 ++++++++++++++++-------------- 11 files changed, 111 insertions(+), 96 deletions(-) diff --git a/common/git.py b/common/git.py index 1ab8b87099..bfb18ce25d 100644 --- a/common/git.py +++ b/common/git.py @@ -24,18 +24,18 @@ def get_branch(cwd: str = None) -> str: @cache -def get_origin() -> str: +def get_origin(cwd: str = None) -> str: try: - local_branch = run_cmd(["git", "name-rev", "--name-only", "HEAD"]) - tracking_remote = run_cmd(["git", "config", "branch." + local_branch + ".remote"]) - return run_cmd(["git", "config", "remote." + tracking_remote + ".url"]) + local_branch = run_cmd(["git", "name-rev", "--name-only", "HEAD"], cwd=cwd) + tracking_remote = run_cmd(["git", "config", "branch." + local_branch + ".remote"], cwd=cwd) + return run_cmd(["git", "config", "remote." + tracking_remote + ".url"], cwd=cwd) except subprocess.CalledProcessError: # Not on a branch, fallback - return run_cmd_default(["git", "config", "--get", "remote.origin.url"]) + return run_cmd_default(["git", "config", "--get", "remote.origin.url"], cwd=cwd) @cache -def get_normalized_origin() -> str: - return get_origin() \ +def get_normalized_origin(cwd: str = None) -> str: + return get_origin(cwd) \ .replace("git@", "", 1) \ .replace(".git", "", 1) \ .replace("https://", "", 1) \ diff --git a/selfdrive/athena/athenad.py b/selfdrive/athena/athenad.py index d228af572c..63fb8669ee 100755 --- a/selfdrive/athena/athenad.py +++ b/selfdrive/athena/athenad.py @@ -32,13 +32,12 @@ from cereal import log from cereal.services import SERVICE_LIST from openpilot.common.api import Api from openpilot.common.file_helpers import CallbackReader -from openpilot.common.git import get_commit, get_normalized_origin, get_short_branch from openpilot.common.params import Params from openpilot.common.realtime import set_core_affinity from openpilot.system.hardware import HARDWARE, PC from openpilot.system.loggerd.xattr_cache import getxattr, setxattr from openpilot.common.swaglog import cloudlog -from openpilot.system.version import get_version +from openpilot.system.version import get_build_metadata from openpilot.system.hardware.hw import Paths @@ -320,11 +319,12 @@ def getMessage(service: str, timeout: int = 1000) -> dict: @dispatcher.add_method def getVersion() -> dict[str, str]: + build_metadata = get_build_metadata() return { - "version": get_version(), - "remote": get_normalized_origin(), - "branch": get_short_branch(), - "commit": get_commit(), + "version": build_metadata.openpilot.version, + "remote": build_metadata.openpilot.git_origin, + "branch": build_metadata.channel, + "commit": build_metadata.openpilot.git_commit, } diff --git a/selfdrive/athena/manage_athenad.py b/selfdrive/athena/manage_athenad.py index 3065bed5c7..5f7a84fd8a 100755 --- a/selfdrive/athena/manage_athenad.py +++ b/selfdrive/athena/manage_athenad.py @@ -7,8 +7,7 @@ from openpilot.common.params import Params from openpilot.selfdrive.manager.process import launcher from openpilot.common.swaglog import cloudlog from openpilot.system.hardware import HARDWARE -from openpilot.common.git import get_commit, get_normalized_origin, get_short_branch -from openpilot.system.version import get_version, is_dirty +from openpilot.system.version import get_build_metadata ATHENA_MGR_PID_PARAM = "AthenadPid" @@ -16,12 +15,14 @@ ATHENA_MGR_PID_PARAM = "AthenadPid" def main(): params = Params() dongle_id = params.get("DongleId").decode('utf-8') + build_metadata = get_build_metadata() + cloudlog.bind_global(dongle_id=dongle_id, - version=get_version(), - origin=get_normalized_origin(), - branch=get_short_branch(), - commit=get_commit(), - dirty=is_dirty(), + version=build_metadata.openpilot.version, + origin=build_metadata.openpilot.git_origin, + branch=build_metadata.channel, + commit=build_metadata.openpilot.git_commit, + dirty=build_metadata.openpilot.git_dirty, device=HARDWARE.get_device_type()) try: diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py index fd8ecc5020..0e2443d330 100644 --- a/selfdrive/car/car_helpers.py +++ b/selfdrive/car/car_helpers.py @@ -4,7 +4,6 @@ from collections.abc import Callable from cereal import car from openpilot.common.params import Params -from openpilot.system.version import is_comma_remote, is_tested_branch from openpilot.selfdrive.car.interfaces import get_interface_attr from openpilot.selfdrive.car.fingerprints import eliminate_incompatible_cars, all_legacy_fingerprint_cars from openpilot.selfdrive.car.vin import get_vin, is_valid_vin, VIN_UNKNOWN @@ -13,6 +12,7 @@ from openpilot.selfdrive.car.mock.values import CAR as MOCK from openpilot.common.swaglog import cloudlog import cereal.messaging as messaging from openpilot.selfdrive.car import gen_empty_fingerprint +from openpilot.system.version import get_build_metadata FRAME_FINGERPRINT = 100 # 1s @@ -20,7 +20,8 @@ EventName = car.CarEvent.EventName def get_startup_event(car_recognized, controller_available, fw_seen): - if is_comma_remote() and is_tested_branch(): + build_metadata = get_build_metadata() + if build_metadata.openpilot.comma_remote and build_metadata.tested_channel: event = EventName.startup else: event = EventName.startupMaster diff --git a/selfdrive/manager/build.py b/selfdrive/manager/build.py index 067e1b5a1e..126d147e9a 100755 --- a/selfdrive/manager/build.py +++ b/selfdrive/manager/build.py @@ -9,7 +9,7 @@ from openpilot.common.spinner import Spinner from openpilot.common.text_window import TextWindow from openpilot.system.hardware import AGNOS from openpilot.common.swaglog import cloudlog, add_file_handler -from openpilot.system.version import is_dirty +from openpilot.system.version import get_build_metadata MAX_CACHE_SIZE = 4e9 if "CI" in os.environ else 2e9 CACHE_DIR = Path("/data/scons_cache" if AGNOS else "/tmp/scons_cache") @@ -86,4 +86,5 @@ def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None: if __name__ == "__main__": spinner = Spinner() spinner.update_progress(0, 100) - build(spinner, is_dirty(), minimal = AGNOS) + build_metadata = get_build_metadata() + build(spinner, build_metadata.openpilot.git_dirty, minimal = AGNOS) diff --git a/selfdrive/manager/manager.py b/selfdrive/manager/manager.py index f30b81861a..2dc79836d1 100755 --- a/selfdrive/manager/manager.py +++ b/selfdrive/manager/manager.py @@ -16,21 +16,20 @@ from openpilot.selfdrive.manager.process import ensure_running from openpilot.selfdrive.manager.process_config import managed_processes from openpilot.selfdrive.athena.registration import register, UNREGISTERED_DONGLE_ID from openpilot.common.swaglog import cloudlog, add_file_handler -from openpilot.common.git import get_commit, get_origin, get_short_branch, get_commit_date -from openpilot.system.version import is_dirty, get_version, \ - get_normalized_origin, terms_version, training_version, \ - is_tested_branch, is_release_branch +from openpilot.system.version import get_build_metadata, terms_version, training_version def manager_init() -> None: save_bootlog() + build_metadata = get_build_metadata() + params = Params() params.clear_all(ParamKeyType.CLEAR_ON_MANAGER_START) params.clear_all(ParamKeyType.CLEAR_ON_ONROAD_TRANSITION) params.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION) - if is_release_branch(): + if build_metadata.release_channel: params.clear_all(ParamKeyType.DEVELOPMENT_ONLY) default_params: list[tuple[str, str | bytes]] = [ @@ -62,15 +61,15 @@ def manager_init() -> None: print("WARNING: failed to make /dev/shm") # set version params - params.put("Version", get_version()) + params.put("Version", build_metadata.openpilot.version) params.put("TermsVersion", terms_version) params.put("TrainingVersion", training_version) - params.put("GitCommit", get_commit()) - params.put("GitCommitDate", get_commit_date()) - params.put("GitBranch", get_short_branch()) - params.put("GitRemote", get_origin()) - params.put_bool("IsTestedBranch", is_tested_branch()) - params.put_bool("IsReleaseBranch", is_release_branch()) + params.put("GitCommit", build_metadata.openpilot.git_commit) + params.put("GitCommitDate", build_metadata.openpilot.git_commit_date) + params.put("GitBranch", build_metadata.channel) + params.put("GitRemote", build_metadata.openpilot.git_origin) + params.put_bool("IsTestedBranch", build_metadata.tested_channel) + params.put_bool("IsReleaseBranch", build_metadata.release_channel) # set dongle id reg_res = register(show_spinner=True) @@ -80,21 +79,21 @@ def manager_init() -> None: serial = params.get("HardwareSerial") raise Exception(f"Registration failed for device {serial}") os.environ['DONGLE_ID'] = dongle_id # Needed for swaglog - os.environ['GIT_ORIGIN'] = get_normalized_origin() # Needed for swaglog - os.environ['GIT_BRANCH'] = get_short_branch() # Needed for swaglog - os.environ['GIT_COMMIT'] = get_commit() # Needed for swaglog + os.environ['GIT_ORIGIN'] = build_metadata.openpilot.git_origin # Needed for swaglog + os.environ['GIT_BRANCH'] = build_metadata.channel # Needed for swaglog + os.environ['GIT_COMMIT'] = build_metadata.openpilot.git_commit # Needed for swaglog - if not is_dirty(): + if not build_metadata.openpilot.git_dirty: os.environ['CLEAN'] = '1' # init logging sentry.init(sentry.SentryProject.SELFDRIVE) cloudlog.bind_global(dongle_id=dongle_id, - version=get_version(), - origin=get_normalized_origin(), - branch=get_short_branch(), - commit=get_commit(), - dirty=is_dirty(), + version=build_metadata.openpilot.version, + origin=build_metadata.openpilot.git_origin, + branch=build_metadata.channel, + commit=build_metadata.openpilot.git_commit, + dirty=build_metadata.openpilot.git_dirty, device=HARDWARE.get_device_type()) # preimport all processes diff --git a/selfdrive/sentry.py b/selfdrive/sentry.py index 889178610f..6abac634b9 100644 --- a/selfdrive/sentry.py +++ b/selfdrive/sentry.py @@ -6,9 +6,8 @@ from sentry_sdk.integrations.threading import ThreadingIntegration from openpilot.common.params import Params from openpilot.selfdrive.athena.registration import is_registered_device from openpilot.system.hardware import HARDWARE, PC -from openpilot.common.git import get_commit, get_branch, get_origin from openpilot.common.swaglog import cloudlog -from openpilot.system.version import get_version, is_comma_remote, is_dirty, is_tested_branch +from openpilot.system.version import get_build_metadata, get_version class SentryProject(Enum): @@ -43,12 +42,13 @@ def set_tag(key: str, value: str) -> None: def init(project: SentryProject) -> bool: + build_metadata = get_build_metadata() # forks like to mess with this, so double check - comma_remote = is_comma_remote() and "commaai" in get_origin() + comma_remote = build_metadata.openpilot.comma_remote and "commaai" in build_metadata.openpilot.git_origin if not comma_remote or not is_registered_device() or PC: return False - env = "release" if is_tested_branch() else "master" + env = "release" if build_metadata.tested_channel else "master" dongle_id = Params().get("DongleId", encoding='utf-8') integrations = [] @@ -63,11 +63,13 @@ def init(project: SentryProject) -> bool: max_value_length=8192, environment=env) + build_metadata = get_build_metadata() + sentry_sdk.set_user({"id": dongle_id}) - sentry_sdk.set_tag("dirty", is_dirty()) - sentry_sdk.set_tag("origin", get_origin()) - sentry_sdk.set_tag("branch", get_branch()) - sentry_sdk.set_tag("commit", get_commit()) + sentry_sdk.set_tag("dirty", build_metadata.openpilot.git_dirty) + sentry_sdk.set_tag("origin", build_metadata.openpilot.git_origin) + sentry_sdk.set_tag("branch", build_metadata.channel) + sentry_sdk.set_tag("commit", build_metadata.openpilot.git_commit) sentry_sdk.set_tag("device", HARDWARE.get_device_type()) if project == SentryProject.SELFDRIVE: diff --git a/selfdrive/statsd.py b/selfdrive/statsd.py index 299aa295d7..4986840e38 100755 --- a/selfdrive/statsd.py +++ b/selfdrive/statsd.py @@ -13,7 +13,7 @@ from openpilot.system.hardware.hw import Paths from openpilot.common.swaglog import cloudlog from openpilot.system.hardware import HARDWARE from openpilot.common.file_helpers import atomic_write_in_dir -from openpilot.system.version import get_normalized_origin, get_short_branch, get_short_version, is_dirty +from openpilot.system.version import get_build_metadata from openpilot.system.loggerd.config import STATS_DIR_FILE_LIMIT, STATS_SOCKET, STATS_FLUSH_TIME_S @@ -86,13 +86,15 @@ def main() -> NoReturn: # initialize stats directory Path(STATS_DIR).mkdir(parents=True, exist_ok=True) + build_metadata = get_build_metadata() + # initialize tags tags = { 'started': False, - 'version': get_short_version(), - 'branch': get_short_branch(), - 'dirty': is_dirty(), - 'origin': get_normalized_origin(), + 'version': build_metadata.openpilot.version, + 'branch': build_metadata.channel, + 'dirty': build_metadata.openpilot.git_dirty, + 'origin': build_metadata.openpilot.git_origin, 'deviceType': HARDWARE.get_device_type(), } diff --git a/selfdrive/tombstoned.py b/selfdrive/tombstoned.py index f1b8c88083..2c99c7eafe 100755 --- a/selfdrive/tombstoned.py +++ b/selfdrive/tombstoned.py @@ -11,8 +11,8 @@ from typing import NoReturn import openpilot.selfdrive.sentry as sentry from openpilot.system.hardware.hw import Paths -from openpilot.common.git import get_commit from openpilot.common.swaglog import cloudlog +from openpilot.system.version import get_build_metadata MAX_SIZE = 1_000_000 * 100 # allow up to 100M MAX_TOMBSTONE_FN_LEN = 62 # 85 - 23 ("/crash/") @@ -124,7 +124,9 @@ def report_tombstone_apport(fn): clean_path = path.replace('/', '_') date = datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S") - new_fn = f"{date}_{(get_commit() or 'nocommit')[:8]}_{safe_fn(clean_path)}"[:MAX_TOMBSTONE_FN_LEN] + build_metadata = get_build_metadata() + + new_fn = f"{date}_{(build_metadata.openpilot.git_commit or 'nocommit')[:8]}_{safe_fn(clean_path)}"[:MAX_TOMBSTONE_FN_LEN] crashlog_dir = os.path.join(Paths.log_root(), "crash") os.makedirs(crashlog_dir, exist_ok=True) diff --git a/selfdrive/updated/updated.py b/selfdrive/updated/updated.py index b6b395f254..3a710ba02f 100755 --- a/selfdrive/updated/updated.py +++ b/selfdrive/updated/updated.py @@ -19,7 +19,7 @@ from openpilot.common.time import system_time_valid from openpilot.system.hardware import AGNOS, HARDWARE from openpilot.common.swaglog import cloudlog from openpilot.selfdrive.controls.lib.alertmanager import set_offroad_alert -from openpilot.system.version import is_tested_branch +from openpilot.system.version import get_build_metadata LOCK_FILE = os.getenv("UPDATER_LOCK_FILE", "/tmp/safe_staging_overlay.lock") STAGING_ROOT = os.getenv("UPDATER_STAGING_ROOT", "/data/safe_staging") @@ -325,8 +325,9 @@ class Updater: now = datetime.datetime.utcnow() dt = now - last_update + build_metadata = get_build_metadata() if failed_count > 15 and exception is not None and self.has_internet: - if is_tested_branch(): + if build_metadata.tested_channel: extra_text = "Ensure the software is correctly installed. Uninstall and re-install if this error persists." else: extra_text = exception diff --git a/system/version.py b/system/version.py index 8866026152..e3b07148e3 100755 --- a/system/version.py +++ b/system/version.py @@ -32,31 +32,13 @@ def get_release_notes(path: str = BASEDIR) -> str: return f.read().split('\n\n', 1)[0] -@cache -def get_short_version() -> str: - return get_version().split('-')[0] - @cache def is_prebuilt() -> bool: return os.path.exists(os.path.join(BASEDIR, 'prebuilt')) @cache -def is_comma_remote() -> bool: - # note to fork maintainers, this is used for release metrics. please do not - # touch this to get rid of the orange startup alert. there's better ways to do that - return get_normalized_origin() == "github.com/commaai/openpilot" - -@cache -def is_tested_branch() -> bool: - return get_short_branch() in TESTED_BRANCHES - -@cache -def is_release_branch() -> bool: - return get_short_branch() in RELEASE_BRANCHES - -@cache -def is_dirty() -> bool: +def is_dirty(cwd: str = None) -> bool: origin = get_origin() branch = get_branch() if not origin or not branch: @@ -68,11 +50,11 @@ def is_dirty() -> bool: if not is_prebuilt(): # This is needed otherwise touched files might show up as modified try: - subprocess.check_call(["git", "update-index", "--refresh"]) + subprocess.check_call(["git", "update-index", "--refresh"], cwd=cwd) except subprocess.CalledProcessError: pass - dirty = (subprocess.call(["git", "diff-index", "--quiet", branch, "--"]) != 0) + dirty = (subprocess.call(["git", "diff-index", "--quiet", branch, "--"], cwd=cwd) != 0) except subprocess.CalledProcessError: cloudlog.exception("git subprocess failed while checking dirty") dirty = True @@ -85,6 +67,19 @@ class OpenpilotMetadata: version: str release_notes: str git_commit: str + git_origin: str + git_commit_date: str + git_dirty: bool + + @property + def short_version(self) -> str: + return self.version.split('-')[0] + + @property + def comma_remote(self) -> bool: + # note to fork maintainers, this is used for release metrics. please do not + # touch this to get rid of the orange startup alert. there's better ways to do that + return self.git_origin == "github.com/commaai/openpilot" @dataclass(frozen=True) @@ -92,9 +87,17 @@ class BuildMetadata: channel: str openpilot: OpenpilotMetadata + @property + def tested_channel(self) -> bool: + return self.channel in TESTED_BRANCHES + + @property + def release_channel(self) -> bool: + return self.channel in RELEASE_BRANCHES + -def get_build_metadata(path: str = BASEDIR) -> BuildMetadata | None: +def get_build_metadata(path: str = BASEDIR) -> BuildMetadata: build_metadata_path = pathlib.Path(path) / BUILD_METADATA_FILENAME if build_metadata_path.exists(): @@ -105,14 +108,25 @@ def get_build_metadata(path: str = BASEDIR) -> BuildMetadata | None: version = openpilot_metadata.get("version", "unknown") release_notes = openpilot_metadata.get("release_notes", "unknown") git_commit = openpilot_metadata.get("git_commit", "unknown") - return BuildMetadata(channel, OpenpilotMetadata(version, release_notes, git_commit)) + git_origin = openpilot_metadata.get("git_origin", "unknown") + git_commit_date = openpilot_metadata.get("git_commit_date", "unknown") + return BuildMetadata(channel, + OpenpilotMetadata( + version=version, + release_notes=release_notes, + git_commit=git_commit, + git_origin=git_origin, + git_commit_date=git_commit_date, + git_dirty=False)) git_folder = pathlib.Path(path) / ".git" if git_folder.exists(): - return BuildMetadata(get_short_branch(path), OpenpilotMetadata(get_version(path), get_release_notes(path), get_commit(path))) + return BuildMetadata(get_short_branch(path), OpenpilotMetadata(get_version(path), get_release_notes(path), get_commit(path), \ + get_normalized_origin(path), get_commit_date(path), is_dirty(path))) - return None + cloudlog.exception("unable to get build metadata") + raise Exception("invalid build metadata") if __name__ == "__main__": @@ -122,12 +136,4 @@ if __name__ == "__main__": params.put("TermsVersion", terms_version) params.put("TrainingVersion", training_version) - print(f"Dirty: {is_dirty()}") - print(f"Version: {get_version()}") - print(f"Short version: {get_short_version()}") - print(f"Origin: {get_origin()}") - print(f"Normalized origin: {get_normalized_origin()}") - print(f"Branch: {get_branch()}") - print(f"Short branch: {get_short_branch()}") - print(f"Prebuilt: {is_prebuilt()}") - print(f"Commit date: {get_commit_date()}") + print(get_build_metadata()) From ce8dac9b4d346108f3cb00471f992ca9a930d70e Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Thu, 21 Mar 2024 00:25:21 +0000 Subject: [PATCH 5/7] fix normailzed --- selfdrive/manager/manager.py | 4 ++-- selfdrive/statsd.py | 2 +- system/version.py | 14 +++++++++++--- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/selfdrive/manager/manager.py b/selfdrive/manager/manager.py index 2dc79836d1..b390bf50bc 100755 --- a/selfdrive/manager/manager.py +++ b/selfdrive/manager/manager.py @@ -79,7 +79,7 @@ def manager_init() -> None: serial = params.get("HardwareSerial") raise Exception(f"Registration failed for device {serial}") os.environ['DONGLE_ID'] = dongle_id # Needed for swaglog - os.environ['GIT_ORIGIN'] = build_metadata.openpilot.git_origin # Needed for swaglog + os.environ['GIT_ORIGIN'] = build_metadata.openpilot.git_normalized_origin # Needed for swaglog os.environ['GIT_BRANCH'] = build_metadata.channel # Needed for swaglog os.environ['GIT_COMMIT'] = build_metadata.openpilot.git_commit # Needed for swaglog @@ -90,7 +90,7 @@ def manager_init() -> None: sentry.init(sentry.SentryProject.SELFDRIVE) cloudlog.bind_global(dongle_id=dongle_id, version=build_metadata.openpilot.version, - origin=build_metadata.openpilot.git_origin, + origin=build_metadata.openpilot.git_normalized_origin, branch=build_metadata.channel, commit=build_metadata.openpilot.git_commit, dirty=build_metadata.openpilot.git_dirty, diff --git a/selfdrive/statsd.py b/selfdrive/statsd.py index 4986840e38..9a10b3a5be 100755 --- a/selfdrive/statsd.py +++ b/selfdrive/statsd.py @@ -94,7 +94,7 @@ def main() -> NoReturn: 'version': build_metadata.openpilot.version, 'branch': build_metadata.channel, 'dirty': build_metadata.openpilot.git_dirty, - 'origin': build_metadata.openpilot.git_origin, + 'origin': build_metadata.openpilot.git_normalized_origin, 'deviceType': HARDWARE.get_device_type(), } diff --git a/system/version.py b/system/version.py index e3b07148e3..deaa6b26ae 100755 --- a/system/version.py +++ b/system/version.py @@ -9,7 +9,7 @@ import subprocess from openpilot.common.basedir import BASEDIR from openpilot.common.swaglog import cloudlog from openpilot.common.utils import cache -from openpilot.common.git import get_commit, get_origin, get_branch, get_short_branch, get_normalized_origin, get_commit_date +from openpilot.common.git import get_commit, get_origin, get_branch, get_short_branch, get_commit_date RELEASE_BRANCHES = ['release3-staging', 'release3', 'nightly'] @@ -79,7 +79,15 @@ class OpenpilotMetadata: def comma_remote(self) -> bool: # note to fork maintainers, this is used for release metrics. please do not # touch this to get rid of the orange startup alert. there's better ways to do that - return self.git_origin == "github.com/commaai/openpilot" + return self.git_normalized_origin == "github.com/commaai/openpilot" + + @property + def git_normalized_origin(self) -> str: + return self.git_origin \ + .replace("git@", "", 1) \ + .replace(".git", "", 1) \ + .replace("https://", "", 1) \ + .replace(":", "/", 1) @dataclass(frozen=True) @@ -123,7 +131,7 @@ def get_build_metadata(path: str = BASEDIR) -> BuildMetadata: if git_folder.exists(): return BuildMetadata(get_short_branch(path), OpenpilotMetadata(get_version(path), get_release_notes(path), get_commit(path), \ - get_normalized_origin(path), get_commit_date(path), is_dirty(path))) + get_origin(path), get_commit_date(path), is_dirty(path))) cloudlog.exception("unable to get build metadata") raise Exception("invalid build metadata") From 7e9b351e7d9c84bd0c992efe9d1a34ab55c1a55f Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Thu, 21 Mar 2024 00:26:09 +0000 Subject: [PATCH 6/7] also normalized --- selfdrive/athena/athenad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfdrive/athena/athenad.py b/selfdrive/athena/athenad.py index 63fb8669ee..989e284e74 100755 --- a/selfdrive/athena/athenad.py +++ b/selfdrive/athena/athenad.py @@ -322,7 +322,7 @@ def getVersion() -> dict[str, str]: build_metadata = get_build_metadata() return { "version": build_metadata.openpilot.version, - "remote": build_metadata.openpilot.git_origin, + "remote": build_metadata.openpilot.git_normalized_origin, "branch": build_metadata.channel, "commit": build_metadata.openpilot.git_commit, } From 96d13f6e2f4907621dc670da82f78639829f8ed2 Mon Sep 17 00:00:00 2001 From: Justin Newberry Date: Thu, 21 Mar 2024 00:26:27 +0000 Subject: [PATCH 7/7] and here --- selfdrive/athena/manage_athenad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfdrive/athena/manage_athenad.py b/selfdrive/athena/manage_athenad.py index 5f7a84fd8a..a15f18eabf 100755 --- a/selfdrive/athena/manage_athenad.py +++ b/selfdrive/athena/manage_athenad.py @@ -19,7 +19,7 @@ def main(): cloudlog.bind_global(dongle_id=dongle_id, version=build_metadata.openpilot.version, - origin=build_metadata.openpilot.git_origin, + origin=build_metadata.openpilot.git_normalized_origin, branch=build_metadata.channel, commit=build_metadata.openpilot.git_commit, dirty=build_metadata.openpilot.git_dirty,