compatibility docs: print diff from PR (#24941)

* print docs diff

* revert car changes

* cause a diff

* temp so it works

* text diff

* tier inline is a bit too much

* comments

* fix

* use paths

* fix

* temp

* temp

* diff

* fix

* remove something

* more text diff

* Delete comment if outdated

* Smaller diff

* remove

* no diff

* Don't try to run on fork PRs

* cause some errors

* Fix

* Fix

* Doesn't support env in job if, only step if

* in case file was moved, don't throw error

* See if this does what I think it does

* See if this does what I think it does

* should work

* change something

* revert

* uncomment

* no comment

* this shouldn't fail

* rename to base

* Remove true

* Remove other true

.github/workflows/selfdrive_tests.yaml

@@ -10,6 +10,7 @@ env:
   CL_BASE_IMAGE: openpilot-base-cl
   DOCKER_REGISTRY: ghcr.io/commaai
   AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}
+  HAS_AZURE_TOKEN: $AZURE_TOKEN != ''
 
   DOCKER_LOGIN: docker login ghcr.io -u adeebshihadeh -p ${{ secrets.CONTAINER_TOKEN }}
   BUILD: |
@@ -17,12 +18,12 @@ env:
       docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
       docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
 
-  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache $BASE_IMAGE /bin/sh -c
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
 
   BUILD_CL: |
       docker pull $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest || true
       docker build --cache-from $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $CL_BASE_IMAGE:latest -f Dockerfile.openpilot_base_cl .
-  RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache $CL_BASE_IMAGE /bin/sh -c
+  RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
 
   UNIT_TEST: coverage run --append -m unittest discover
 
@@ -365,7 +366,7 @@ jobs:
         name: process_replay_diff.txt
         path: selfdrive/test/process_replay/diff.txt
     - name: Upload reference logs
-      if: ${{ failure() && github.event_name == 'pull_request' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
+      if: ${{ failure() && github.event_name == 'pull_request' && github.repository == 'commaai/openpilot' && env.HAS_AZURE_TOKEN }}
       run: |
         ${{ env.RUN }} "scons -j$(nproc) && \
                         CI=1 AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
@@ -510,3 +511,67 @@ jobs:
       run: |
         $DOCKER_LOGIN
         docker push $DOCKER_REGISTRY/openpilot-docs:latest
+
+  car_docs_diff:
+    name: comment on PR with car docs diff
+    runs-on: ubuntu-20.04
+    timeout-minutes: 50
+    if: github.event_name == 'pull_request'
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+          ref: ${{ github.event.pull_request.base.ref }}
+      - name: Cache scons
+        id: scons-cache
+        # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
+        uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
+        env:
+          CACHE_SKIP_SAVE: true
+        with:
+          path: /tmp/scons_cache
+          key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
+          restore-keys: |
+            scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
+            scons-
+      - name: Build Docker image
+        run: eval "$BUILD"
+      - name: Get base car info
+        run: |
+          ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_info.py --path /tmp/openpilot_cache/base_car_info"
+          sudo chown -R $USER:$USER ${{ github.workspace }}
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Save car docs diff
+        id: save_diff
+        run: |
+          ${{ env.RUN }} "scons -j$(nproc)"
+          output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_info")
+          output="${output//$'\n'/'%0A'}"
+          echo "::set-output name=diff::$output"
+      - name: Find comment
+        if: env.HAS_AZURE_TOKEN
+        uses: peter-evans/find-comment@v1
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          body-includes: This PR makes changes to
+      - name: Update comment
+        if: steps.save_diff.outputs.diff != '' && env.HAS_AZURE_TOKEN
+        uses: peter-evans/create-or-update-comment@v1
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body: "${{ steps.save_diff.outputs.diff }}"
+          edit-mode: replace
+      - name: Delete comment
+        if: steps.fc.outputs.comment-id != '' && steps.save_diff.outputs.diff == '' && env.HAS_AZURE_TOKEN
+        uses: actions/github-script@v6
+        with:
+          script: |
+            github.rest.issues.deleteComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              comment_id: ${{ steps.fc.outputs.comment-id }}
+            })

selfdrive/debug/dump_car_info.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+import argparse
+import pickle
+
+from selfdrive.car.docs import get_all_car_info
+
+
+def dump_car_info(path):
+  with open(path, 'wb') as f:
+    pickle.dump(get_all_car_info(), f)
+  print(f'Dumping car info to {path}')
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--path", required=True)
+  args = parser.parse_args()
+  dump_car_info(args.path)

selfdrive/debug/print_docs_diff.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+import argparse
+import pickle
+
+from selfdrive.car.docs import get_all_car_info
+from selfdrive.car.docs_definitions import Column
+
+STAR_ICON = '<a href="##"><img valign="top" src="https://raw.githubusercontent.com/commaai/openpilot/master/docs/assets/icon-star-{}.svg" width="22" /></a>'
+COLUMNS = "|" + "|".join([column.value for column in Column]) + "|"
+COLUMN_HEADER = "|---|---|---|:---:|:---:|:---:|:---:|:---:|"
+ARROW_SYMBOL = "➡️"
+
+
+def load_base_car_info(path):
+  with open(path, "rb") as f:
+    return pickle.load(f)
+
+
+def get_star_diff(base_car, new_car):
+  return [column for column, value in base_car.row.items() if value != new_car.row[column]]
+
+
+def format_row(builder):
+  return "|" + "|".join(builder) + "|"
+
+
+def print_car_info_diff(path):
+  base_car_info = {f"{i.make} {i.model}": i for i in load_base_car_info(path)}
+  new_car_info = {f"{i.make} {i.model}": i for i in get_all_car_info()}
+
+  tier_changes = []
+  star_changes = []
+  removals = []
+  additions = []
+
+  # Changes (tier + stars)
+  for base_car_model, base_car in base_car_info.items():
+    if base_car_model not in new_car_info:
+      continue
+
+    new_car = new_car_info[base_car_model]
+
+    # Tier changes
+    if base_car.tier != new_car.tier:
+      tier_changes.append(f"- Tier for {base_car.make} {base_car.model} changed! ({base_car.tier.name.title()} {ARROW_SYMBOL} {new_car.tier.name.title()})")
+
+    # Star changes
+    diff = get_star_diff(base_car, new_car)
+    if not len(diff):
+      continue
+
+    row_builder = []
+    for column in list(Column):
+      if column not in diff:
+        row_builder.append(new_car.get_column(column, STAR_ICON, "{}"))
+      else:
+        row_builder.append(base_car.get_column(column, STAR_ICON, "{}") + ARROW_SYMBOL + new_car.get_column(column, STAR_ICON, "{}"))
+
+    star_changes.append(format_row(row_builder))
+
+  # Removals
+  for model in set(base_car_info) - set(new_car_info):
+    car_info = base_car_info[model]
+    removals.append(format_row([car_info.get_column(column, STAR_ICON, "{}") for column in Column]))
+
+  # Additions
+  for model in set(new_car_info) - set(base_car_info):
+    car_info = new_car_info[model]
+    additions.append(format_row([car_info.get_column(column, STAR_ICON, "{}") for column in Column]))
+
+  # Print diff
+  if len(star_changes) or len(tier_changes) or len(removals) or len(additions):
+    markdown_builder = ["### ⚠️ This PR makes changes to [CARS.md](../blob/master/docs/CARS.md) ⚠️"]
+
+    for title, category in (("## 🏅 Tier Changes", tier_changes), ("## 🔀 Star Changes", star_changes), ("## ❌ Removed", removals), ("## ➕ Added", additions)):
+      if len(category):
+        markdown_builder.append(title)
+        if "Tier" not in title:
+          markdown_builder.append(COLUMNS)
+          markdown_builder.append(COLUMN_HEADER)
+        markdown_builder.extend(category)
+
+    print("\n".join(markdown_builder))
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--path", required=True)
+  args = parser.parse_args()
+  print_car_info_diff(args.path)