From 37e86df41e2a411ac5db791a8315c29f543e5155 Mon Sep 17 00:00:00 2001
From: Adeeb Shihadeh <adeebshihadeh@gmail.com>
Date: Thu, 24 Apr 2025 18:16:18 -0700
Subject: [PATCH] CI: comment reporter links on PRs (#35066)

* comment

* Revert "Tomb Raider 4 (#35058)"

This reverts commit 756aebd39f378a6a1db2826b4044290b391cc03c.

* no dpeth

* Reapply "Tomb Raider 4 (#35058)"

This reverts commit 7143835b3d075271e86fb181e08f32db7720c6f8.

* no forks
---
 .github/workflows/model_review.yaml | 42 +++++++++++++++++++++++++++++
 scripts/reporter.py                 | 31 +++++++++++++++++++++
 2 files changed, 73 insertions(+)
 create mode 100644 .github/workflows/model_review.yaml
 create mode 100755 scripts/reporter.py

diff --git a/.github/workflows/model_review.yaml b/.github/workflows/model_review.yaml
new file mode 100644
index 0000000000..0e1825864c
--- /dev/null
+++ b/.github/workflows/model_review.yaml
@@ -0,0 +1,42 @@
+name: "model review"
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+    paths:
+      - 'selfdrive/modeld/models/*.onnx'
+  workflow_dispatch:
+
+jobs:
+  comment:
+    permissions:
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu-latest
+    if: github.repository == 'commaai/openpilot'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Checkout master
+        uses: actions/checkout@v4
+        with:
+          ref: master
+          path: base
+      - run: git lfs pull
+      - run: cd base && git lfs pull
+
+      - run: pip install onnx
+
+      - name: scripts/reporter.py
+        id: report
+        run: |
+          echo "content<<EOF" >> $GITHUB_OUTPUT
+          echo "## Model Review" >> $GITHUB_OUTPUT
+          MASTER_PATH=${{ github.workspace }}/base python scripts/reporter.py >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
+      - name: Post model report comment
+        uses: marocchino/sticky-pull-request-comment@baa7203ed60924babbe5dcd0ac8eae3b66ec5e16
+        with:
+          header: model-review
+          message: ${{ steps.report.outputs.content }}
\ No newline at end of file
diff --git a/scripts/reporter.py b/scripts/reporter.py
new file mode 100755
index 0000000000..903fcc8911
--- /dev/null
+++ b/scripts/reporter.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+import os
+import glob
+import onnx
+
+BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../"))
+MASTER_PATH = os.getenv("MASTER_PATH", BASEDIR)
+MODEL_PATH = "/selfdrive/modeld/models/"
+
+def get_checkpoint(f):
+  model = onnx.load(f)
+  metadata = {prop.key: prop.value for prop in model.metadata_props}
+  return metadata['model_checkpoint'].split('/')[0]
+
+if __name__ == "__main__":
+  print("| | master | PR branch |")
+  print("|-| -----  | --------- |")
+
+  for f in glob.glob(BASEDIR + MODEL_PATH + "/*.onnx"):
+    # TODO: add checkpoint to DM
+    if "dmonitoring" in f:
+      continue
+
+    fn = os.path.basename(f)
+    master = get_checkpoint(MASTER_PATH + MODEL_PATH + fn)
+    pr = get_checkpoint(BASEDIR + MODEL_PATH + fn)
+    print(
+      "|", fn, "|",
+      f"[{master}](https://reporter.comma.life/experiment/{master})", "|",
+      f"[{pr}](https://reporter.comma.life/experiment/{pr})", "|"
+    )