use build_metadata everywhere we used to use get_version, get_commit, etc (#31941)

* use build_metadata

* fix normailzed

* also normalized

* and here

* fix diff

* and that one

* cleanup

selfdrive/athena/athenad.py

@@ -32,13 +32,12 @@ from cereal import log
 from cereal.services import SERVICE_LIST
 from openpilot.common.api import Api
 from openpilot.common.file_helpers import CallbackReader
-from openpilot.common.git import get_commit, get_normalized_origin, get_short_branch
 from openpilot.common.params import Params
 from openpilot.common.realtime import set_core_affinity
 from openpilot.system.hardware import HARDWARE, PC
 from openpilot.system.loggerd.xattr_cache import getxattr, setxattr
 from openpilot.common.swaglog import cloudlog
-from openpilot.system.version import get_version
+from openpilot.system.version import get_build_metadata
 from openpilot.system.hardware.hw import Paths
 
 
@@ -320,11 +319,12 @@ def getMessage(service: str, timeout: int = 1000) -> dict:
 
 @dispatcher.add_method
 def getVersion() -> dict[str, str]:
+  build_metadata = get_build_metadata()
   return {
-    "version": get_version(),
-    "remote": get_normalized_origin(),
-    "branch": get_short_branch(),
-    "commit": get_commit(),
+    "version": build_metadata.openpilot.version,
+    "remote": build_metadata.openpilot.git_normalized_origin,
+    "branch": build_metadata.channel,
+    "commit": build_metadata.openpilot.git_commit,
   }
 
 

selfdrive/athena/manage_athenad.py

@@ -7,8 +7,7 @@ from openpilot.common.params import Params
 from openpilot.selfdrive.manager.process import launcher
 from openpilot.common.swaglog import cloudlog
 from openpilot.system.hardware import HARDWARE
-from openpilot.common.git import get_commit, get_normalized_origin, get_short_branch
-from openpilot.system.version import get_version, is_dirty
+from openpilot.system.version import get_build_metadata
 
 ATHENA_MGR_PID_PARAM = "AthenadPid"
 
@@ -16,12 +15,14 @@ ATHENA_MGR_PID_PARAM = "AthenadPid"
 def main():
   params = Params()
   dongle_id = params.get("DongleId").decode('utf-8')
+  build_metadata = get_build_metadata()
+
   cloudlog.bind_global(dongle_id=dongle_id,
-                       version=get_version(),
-                       origin=get_normalized_origin(),
-                       branch=get_short_branch(),
-                       commit=get_commit(),
-                       dirty=is_dirty(),
+                       version=build_metadata.openpilot.version,
+                       origin=build_metadata.openpilot.git_normalized_origin,
+                       branch=build_metadata.channel,
+                       commit=build_metadata.openpilot.git_commit,
+                       dirty=build_metadata.openpilot.is_dirty,
                        device=HARDWARE.get_device_type())
 
   try:

selfdrive/car/car_helpers.py

@@ -4,7 +4,6 @@ from collections.abc import Callable
 
 from cereal import car
 from openpilot.common.params import Params
-from openpilot.system.version import is_comma_remote, is_tested_branch
 from openpilot.selfdrive.car.interfaces import get_interface_attr
 from openpilot.selfdrive.car.fingerprints import eliminate_incompatible_cars, all_legacy_fingerprint_cars
 from openpilot.selfdrive.car.vin import get_vin, is_valid_vin, VIN_UNKNOWN
@@ -13,6 +12,7 @@ from openpilot.selfdrive.car.mock.values import CAR as MOCK
 from openpilot.common.swaglog import cloudlog
 import cereal.messaging as messaging
 from openpilot.selfdrive.car import gen_empty_fingerprint
+from openpilot.system.version import get_build_metadata
 
 FRAME_FINGERPRINT = 100  # 1s
 
@@ -20,7 +20,8 @@ EventName = car.CarEvent.EventName
 
 
 def get_startup_event(car_recognized, controller_available, fw_seen):
-  if is_comma_remote() and is_tested_branch():
+  build_metadata = get_build_metadata()
+  if build_metadata.openpilot.comma_remote and build_metadata.tested_channel:
     event = EventName.startup
   else:
     event = EventName.startupMaster

selfdrive/manager/build.py

@@ -9,7 +9,7 @@ from openpilot.common.spinner import Spinner
 from openpilot.common.text_window import TextWindow
 from openpilot.system.hardware import AGNOS
 from openpilot.common.swaglog import cloudlog, add_file_handler
-from openpilot.system.version import is_dirty
+from openpilot.system.version import get_build_metadata
 
 MAX_CACHE_SIZE = 4e9 if "CI" in os.environ else 2e9
 CACHE_DIR = Path("/data/scons_cache" if AGNOS else "/tmp/scons_cache")
@@ -86,4 +86,5 @@ def build(spinner: Spinner, dirty: bool = False, minimal: bool = False) -> None:
 if __name__ == "__main__":
   spinner = Spinner()
   spinner.update_progress(0, 100)
-  build(spinner, is_dirty(), minimal = AGNOS)
+  build_metadata = get_build_metadata()
+  build(spinner, build_metadata.openpilot.is_dirty, minimal = AGNOS)

selfdrive/manager/manager.py

@@ -16,21 +16,20 @@ from openpilot.selfdrive.manager.process import ensure_running
 from openpilot.selfdrive.manager.process_config import managed_processes
 from openpilot.selfdrive.athena.registration import register, UNREGISTERED_DONGLE_ID
 from openpilot.common.swaglog import cloudlog, add_file_handler
-from openpilot.common.git import get_commit, get_origin, get_short_branch, get_commit_date
-from openpilot.system.version import is_dirty, get_version, \
-                           get_normalized_origin, terms_version, training_version, \
-                           is_tested_branch, is_release_branch
+from openpilot.system.version import get_build_metadata, terms_version, training_version
 
 
 
 def manager_init() -> None:
   save_bootlog()
 
+  build_metadata = get_build_metadata()
+
   params = Params()
   params.clear_all(ParamKeyType.CLEAR_ON_MANAGER_START)
   params.clear_all(ParamKeyType.CLEAR_ON_ONROAD_TRANSITION)
   params.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION)
-  if is_release_branch():
+  if build_metadata.release_channel:
     params.clear_all(ParamKeyType.DEVELOPMENT_ONLY)
 
   default_params: list[tuple[str, str | bytes]] = [
@@ -62,15 +61,15 @@ def manager_init() -> None:
     print("WARNING: failed to make /dev/shm")
 
   # set version params
-  params.put("Version", get_version())
+  params.put("Version", build_metadata.openpilot.version)
   params.put("TermsVersion", terms_version)
   params.put("TrainingVersion", training_version)
-  params.put("GitCommit", get_commit())
-  params.put("GitCommitDate", get_commit_date())
-  params.put("GitBranch", get_short_branch())
-  params.put("GitRemote", get_origin())
-  params.put_bool("IsTestedBranch", is_tested_branch())
-  params.put_bool("IsReleaseBranch", is_release_branch())
+  params.put("GitCommit", build_metadata.openpilot.git_commit)
+  params.put("GitCommitDate", build_metadata.openpilot.git_commit_date)
+  params.put("GitBranch", build_metadata.channel)
+  params.put("GitRemote", build_metadata.openpilot.git_origin)
+  params.put_bool("IsTestedBranch", build_metadata.tested_channel)
+  params.put_bool("IsReleaseBranch", build_metadata.release_channel)
 
   # set dongle id
   reg_res = register(show_spinner=True)
@@ -80,21 +79,21 @@ def manager_init() -> None:
     serial = params.get("HardwareSerial")
     raise Exception(f"Registration failed for device {serial}")
   os.environ['DONGLE_ID'] = dongle_id  # Needed for swaglog
-  os.environ['GIT_ORIGIN'] = get_normalized_origin() # Needed for swaglog
-  os.environ['GIT_BRANCH'] = get_short_branch() # Needed for swaglog
-  os.environ['GIT_COMMIT'] = get_commit() # Needed for swaglog
+  os.environ['GIT_ORIGIN'] = build_metadata.openpilot.git_normalized_origin # Needed for swaglog
+  os.environ['GIT_BRANCH'] = build_metadata.channel # Needed for swaglog
+  os.environ['GIT_COMMIT'] = build_metadata.openpilot.git_commit # Needed for swaglog
 
-  if not is_dirty():
+  if not build_metadata.openpilot.is_dirty:
     os.environ['CLEAN'] = '1'
 
   # init logging
   sentry.init(sentry.SentryProject.SELFDRIVE)
   cloudlog.bind_global(dongle_id=dongle_id,
-                       version=get_version(),
-                       origin=get_normalized_origin(),
-                       branch=get_short_branch(),
-                       commit=get_commit(),
-                       dirty=is_dirty(),
+                       version=build_metadata.openpilot.version,
+                       origin=build_metadata.openpilot.git_normalized_origin,
+                       branch=build_metadata.channel,
+                       commit=build_metadata.openpilot.git_commit,
+                       dirty=build_metadata.openpilot.is_dirty,
                        device=HARDWARE.get_device_type())
 
   # preimport all processes

selfdrive/sentry.py

@@ -6,9 +6,8 @@ from sentry_sdk.integrations.threading import ThreadingIntegration
 from openpilot.common.params import Params
 from openpilot.selfdrive.athena.registration import is_registered_device
 from openpilot.system.hardware import HARDWARE, PC
-from openpilot.common.git import get_commit, get_branch, get_origin
 from openpilot.common.swaglog import cloudlog
-from openpilot.system.version import get_version, is_comma_remote, is_dirty, is_tested_branch
+from openpilot.system.version import get_build_metadata, get_version
 
 
 class SentryProject(Enum):
@@ -43,12 +42,13 @@ def set_tag(key: str, value: str) -> None:
 
 
 def init(project: SentryProject) -> bool:
+  build_metadata = get_build_metadata()
   # forks like to mess with this, so double check
-  comma_remote = is_comma_remote() and "commaai" in get_origin()
+  comma_remote = build_metadata.openpilot.comma_remote and "commaai" in build_metadata.openpilot.git_origin
   if not comma_remote or not is_registered_device() or PC:
     return False
 
-  env = "release" if is_tested_branch() else "master"
+  env = "release" if build_metadata.tested_channel else "master"
   dongle_id = Params().get("DongleId", encoding='utf-8')
 
   integrations = []
@@ -63,11 +63,13 @@ def init(project: SentryProject) -> bool:
                   max_value_length=8192,
                   environment=env)
 
+  build_metadata = get_build_metadata()
+
   sentry_sdk.set_user({"id": dongle_id})
-  sentry_sdk.set_tag("dirty", is_dirty())
-  sentry_sdk.set_tag("origin", get_origin())
-  sentry_sdk.set_tag("branch", get_branch())
-  sentry_sdk.set_tag("commit", get_commit())
+  sentry_sdk.set_tag("dirty", build_metadata.openpilot.is_dirty)
+  sentry_sdk.set_tag("origin", build_metadata.openpilot.git_origin)
+  sentry_sdk.set_tag("branch", build_metadata.channel)
+  sentry_sdk.set_tag("commit", build_metadata.openpilot.git_commit)
   sentry_sdk.set_tag("device", HARDWARE.get_device_type())
 
   if project == SentryProject.SELFDRIVE:

selfdrive/statsd.py

@@ -13,7 +13,7 @@ from openpilot.system.hardware.hw import Paths
 from openpilot.common.swaglog import cloudlog
 from openpilot.system.hardware import HARDWARE
 from openpilot.common.file_helpers import atomic_write_in_dir
-from openpilot.system.version import get_normalized_origin, get_short_branch, get_short_version, is_dirty
+from openpilot.system.version import get_build_metadata
 from openpilot.system.loggerd.config import STATS_DIR_FILE_LIMIT, STATS_SOCKET, STATS_FLUSH_TIME_S
 
 
@@ -86,13 +86,15 @@ def main() -> NoReturn:
   # initialize stats directory
   Path(STATS_DIR).mkdir(parents=True, exist_ok=True)
 
+  build_metadata = get_build_metadata()
+
   # initialize tags
   tags = {
     'started': False,
-    'version': get_short_version(),
-    'branch': get_short_branch(),
-    'dirty': is_dirty(),
-    'origin': get_normalized_origin(),
+    'version': build_metadata.openpilot.version,
+    'branch': build_metadata.channel,
+    'dirty': build_metadata.openpilot.is_dirty,
+    'origin': build_metadata.openpilot.git_normalized_origin,
     'deviceType': HARDWARE.get_device_type(),
   }
 

selfdrive/tombstoned.py

@@ -11,8 +11,8 @@ from typing import NoReturn
 
 import openpilot.selfdrive.sentry as sentry
 from openpilot.system.hardware.hw import Paths
-from openpilot.common.git import get_commit
 from openpilot.common.swaglog import cloudlog
+from openpilot.system.version import get_build_metadata
 
 MAX_SIZE = 1_000_000 * 100  # allow up to 100M
 MAX_TOMBSTONE_FN_LEN = 62  # 85 - 23 ("<dongle id>/crash/")
@@ -124,7 +124,9 @@ def report_tombstone_apport(fn):
   clean_path = path.replace('/', '_')
   date = datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S")
 
-  new_fn = f"{date}_{(get_commit() or 'nocommit')[:8]}_{safe_fn(clean_path)}"[:MAX_TOMBSTONE_FN_LEN]
+  build_metadata = get_build_metadata()
+
+  new_fn = f"{date}_{(build_metadata.openpilot.git_commit or 'nocommit')[:8]}_{safe_fn(clean_path)}"[:MAX_TOMBSTONE_FN_LEN]
 
   crashlog_dir = os.path.join(Paths.log_root(), "crash")
   os.makedirs(crashlog_dir, exist_ok=True)

selfdrive/updated/updated.py

@@ -19,7 +19,7 @@ from openpilot.common.time import system_time_valid
 from openpilot.system.hardware import AGNOS, HARDWARE
 from openpilot.common.swaglog import cloudlog
 from openpilot.selfdrive.controls.lib.alertmanager import set_offroad_alert
-from openpilot.system.version import is_tested_branch
+from openpilot.system.version import get_build_metadata
 
 LOCK_FILE = os.getenv("UPDATER_LOCK_FILE", "/tmp/safe_staging_overlay.lock")
 STAGING_ROOT = os.getenv("UPDATER_STAGING_ROOT", "/data/safe_staging")
@@ -325,8 +325,9 @@ class Updater:
 
     now = datetime.datetime.utcnow()
     dt = now - last_update
+    build_metadata = get_build_metadata()
     if failed_count > 15 and exception is not None and self.has_internet:
-      if is_tested_branch():
+      if build_metadata.tested_channel:
         extra_text = "Ensure the software is correctly installed. Uninstall and re-install if this error persists."
       else:
         extra_text = exception

system/version.py

@@ -9,7 +9,7 @@ import subprocess
 from openpilot.common.basedir import BASEDIR
 from openpilot.common.swaglog import cloudlog
 from openpilot.common.utils import cache
-from openpilot.common.git import get_commit, get_origin, get_branch, get_short_branch, get_normalized_origin, get_commit_date
+from openpilot.common.git import get_commit, get_origin, get_branch, get_short_branch, get_commit_date
 
 
 RELEASE_BRANCHES = ['release3-staging', 'release3', 'nightly']
@@ -32,33 +32,15 @@ def get_release_notes(path: str = BASEDIR) -> str:
     return f.read().split('\n\n', 1)[0]
 
 
-@cache
-def get_short_version() -> str:
-  return get_version().split('-')[0]
-
 @cache
 def is_prebuilt(path: str = BASEDIR) -> bool:
   return os.path.exists(os.path.join(path, 'prebuilt'))
 
 
-@cache
-def is_comma_remote() -> bool:
-  # note to fork maintainers, this is used for release metrics. please do not
-  # touch this to get rid of the orange startup alert. there's better ways to do that
-  return get_normalized_origin() == "github.com/commaai/openpilot"
-
-@cache
-def is_tested_branch() -> bool:
-  return get_short_branch() in TESTED_BRANCHES
-
-@cache
-def is_release_branch() -> bool:
-  return get_short_branch() in RELEASE_BRANCHES
-
 @cache
 def is_dirty(cwd: str = BASEDIR) -> bool:
-  origin = get_origin(cwd)
-  branch = get_branch(cwd)
+  origin = get_origin()
+  branch = get_branch()
   if not origin or not branch:
     return True
 
@@ -85,6 +67,27 @@ class OpenpilotMetadata:
   version: str
   release_notes: str
   git_commit: str
+  git_origin: str
+  git_commit_date: str
+  is_dirty: bool  # whether there are local changes
+
+  @property
+  def short_version(self) -> str:
+    return self.version.split('-')[0]
+
+  @property
+  def comma_remote(self) -> bool:
+    # note to fork maintainers, this is used for release metrics. please do not
+    # touch this to get rid of the orange startup alert. there's better ways to do that
+    return self.git_normalized_origin == "github.com/commaai/openpilot"
+
+  @property
+  def git_normalized_origin(self) -> str:
+    return self.git_origin \
+      .replace("git@", "", 1) \
+      .replace(".git", "", 1) \
+      .replace("https://", "", 1) \
+      .replace(":", "/", 1)
 
 
 @dataclass(frozen=True)
@@ -92,9 +95,17 @@ class BuildMetadata:
   channel: str
   openpilot: OpenpilotMetadata
 
+  @property
+  def tested_channel(self) -> bool:
+    return self.channel in TESTED_BRANCHES
+
+  @property
+  def release_channel(self) -> bool:
+    return self.channel in RELEASE_BRANCHES
+
 
 
-def get_build_metadata(path: str = BASEDIR) -> BuildMetadata | None:
+def get_build_metadata(path: str = BASEDIR) -> BuildMetadata:
   build_metadata_path = pathlib.Path(path) / BUILD_METADATA_FILENAME
 
   if build_metadata_path.exists():
@@ -105,14 +116,31 @@ def get_build_metadata(path: str = BASEDIR) -> BuildMetadata | None:
     version = openpilot_metadata.get("version", "unknown")
     release_notes = openpilot_metadata.get("release_notes", "unknown")
     git_commit = openpilot_metadata.get("git_commit", "unknown")
-    return BuildMetadata(channel, OpenpilotMetadata(version, release_notes, git_commit))
+    git_origin = openpilot_metadata.get("git_origin", "unknown")
+    git_commit_date = openpilot_metadata.get("git_commit_date", "unknown")
+    return BuildMetadata(channel,
+              OpenpilotMetadata(
+                version=version,
+                release_notes=release_notes,
+                git_commit=git_commit,
+                git_origin=git_origin,
+                git_commit_date=git_commit_date,
+                is_dirty=False))
 
   git_folder = pathlib.Path(path) / ".git"
 
   if git_folder.exists():
-    return BuildMetadata(get_short_branch(path), OpenpilotMetadata(get_version(path), get_release_notes(path), get_commit(path)))
+    return BuildMetadata(get_short_branch(path),
+                    OpenpilotMetadata(
+                      version=get_version(path),
+                      release_notes=get_release_notes(path),
+                      git_commit=get_commit(path),
+                      git_origin=get_origin(path),
+                      git_commit_date=get_commit_date(path),
+                      is_dirty=is_dirty(path)))
 
-  return None
+  cloudlog.exception("unable to get build metadata")
+  raise Exception("invalid build metadata")
 
 
 if __name__ == "__main__":
@@ -122,12 +150,4 @@ if __name__ == "__main__":
   params.put("TermsVersion", terms_version)
   params.put("TrainingVersion", training_version)
 
-  print(f"Dirty: {is_dirty()}")
-  print(f"Version: {get_version()}")
-  print(f"Short version: {get_short_version()}")
-  print(f"Origin: {get_origin()}")
-  print(f"Normalized origin: {get_normalized_origin()}")
-  print(f"Branch: {get_branch()}")
-  print(f"Short branch: {get_short_branch()}")
-  print(f"Prebuilt: {is_prebuilt()}")
-  print(f"Commit date: {get_commit_date()}")
+  print(get_build_metadata())