raylib: ui diff test (#36213)

* add raylib ui test

* match qt

* exe

* vibing is epic

* this is epic

* format

* add more settings

* fix to actually use raylib

* add kb

* global

* pair

* rm cmts

* show event

* this is so stupid

clean up

* clean up

* rename dir

* clean up

* no more vibe

* rm

* ugh it's always slightly different for no reason

* nvm region is actually broken

* 1l

.github/workflows/raylib_ui_preview.yaml

@@ -0,0 +1,174 @@
+name: "raylib ui preview"
+on:
+  push:
+    branches:
+      - master
+  pull_request_target:
+    types: [assigned, opened, synchronize, reopened, edited]
+    branches:
+      - 'master'
+    paths:
+      - 'selfdrive/ui/**'
+      - 'system/ui/**'
+  workflow_dispatch:
+
+env:
+  UI_JOB_NAME: "Create raylib UI Report"
+  REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
+  SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}
+  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}-raylib-ui"
+
+jobs:
+  preview:
+    if: github.repository == 'commaai/openpilot'
+    name: preview
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: read
+      pull-requests: write
+      actions: read
+    steps:
+      - name: Waiting for ui generation to start
+        run: sleep 30
+
+      - name: Waiting for ui generation to end
+        uses: lewagon/wait-on-check-action@v1.3.4
+        with:
+          ref: ${{ env.SHA }}
+          check-name: ${{ env.UI_JOB_NAME }}
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          allowed-conclusions: success
+          wait-interval: 20
+
+      - name: Getting workflow run ID
+        id: get_run_id
+        run: |
+          echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
+
+      - name: Getting proposed ui
+        id: download-artifact
+        uses: dawidd6/action-download-artifact@v6
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          run_id: ${{ steps.get_run_id.outputs.run_id }}
+          search_artifacts: true
+          name: raylib-report-${{ env.REPORT_NAME }}
+          path: ${{ github.workspace }}/pr_ui
+
+      - name: Getting master ui
+        uses: actions/checkout@v4
+        with:
+          repository: commaai/ci-artifacts
+          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
+          path: ${{ github.workspace }}/master_ui_raylib
+          ref: openpilot_master_ui_raylib
+
+      - name: Saving new master ui
+        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+        working-directory: ${{ github.workspace }}/master_ui_raylib
+        run: |
+          git checkout --orphan=new_master_ui_raylib
+          git rm -rf *
+          git branch -D openpilot_master_ui_raylib
+          git branch -m openpilot_master_ui_raylib
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+          mv ${{ github.workspace }}/pr_ui/*.png .
+          git add .
+          git commit -m "raylib screenshots for commit ${{ env.SHA }}"
+          git push origin openpilot_master_ui_raylib --force
+
+      - name: Finding diff
+        if: github.event_name == 'pull_request_target'
+        id: find_diff
+        run: >-
+          sudo apt-get update && sudo apt-get install -y imagemagick
+
+          scenes=$(find ${{ github.workspace }}/pr_ui/*.png -type f -printf "%f\n" | cut -d '.' -f 1 | grep -v 'pair_device')
+          A=($scenes)
+
+          DIFF=""
+          TABLE="<details><summary>All Screenshots</summary>"
+          TABLE="${TABLE}<table>"
+
+          for ((i=0; i<${#A[*]}; i=i+1));
+          do
+            # Check if the master file exists
+            if [ ! -f "${{ github.workspace }}/master_ui_raylib/${A[$i]}.png" ]; then
+              # This is a new file in PR UI that doesn't exist in master
+              DIFF="${DIFF}<details open>"
+              DIFF="${DIFF}<summary>${A[$i]} : \$\${\\color{cyan}\\text{NEW}}\$\$</summary>"
+              DIFF="${DIFF}<table>"
+
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+              DIFF="${DIFF}</tr>"
+
+              DIFF="${DIFF}</table>"
+              DIFF="${DIFF}</details>"
+            elif ! compare -fuzz 2% -highlight-color DeepSkyBlue1 -lowlight-color Black -compose Src ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png; then
+              convert ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png -transparent black mask.png
+              composite mask.png ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png composite_diff.png
+              convert -delay 100 ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png composite_diff.png -loop 0 ${{ github.workspace }}/pr_ui/${A[$i]}_diff.gif
+
+              mv ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_master_ref.png
+
+              DIFF="${DIFF}<details open>"
+              DIFF="${DIFF}<summary>${A[$i]} : \$\${\\color{red}\\text{DIFFERENT}}\$\$</summary>"
+              DIFF="${DIFF}<table>"
+
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> master <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_master_ref.png\"> </td>"
+              DIFF="${DIFF}  <td> proposed <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+              DIFF="${DIFF}</tr>"
+
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> diff <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.png\"> </td>"
+              DIFF="${DIFF}  <td> composite diff <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.gif\"> </td>"
+              DIFF="${DIFF}</tr>"
+
+              DIFF="${DIFF}</table>"
+              DIFF="${DIFF}</details>"
+            else
+              rm -f ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png
+            fi
+
+            INDEX=$(($i % 2))
+            if [[ $INDEX -eq 0 ]]; then
+              TABLE="${TABLE}<tr>"
+            fi
+            TABLE="${TABLE}  <td> <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+            if [[ $INDEX -eq 1 || $(($i + 1)) -eq ${#A[*]} ]]; then
+              TABLE="${TABLE}</tr>"
+            fi
+          done
+
+          TABLE="${TABLE}</table></details>"
+
+          echo "DIFF=$DIFF$TABLE" >> "$GITHUB_OUTPUT"
+
+      - name: Saving proposed ui
+        if: github.event_name == 'pull_request_target'
+        working-directory: ${{ github.workspace }}/master_ui_raylib
+        run: |
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+          git checkout --orphan=${{ env.BRANCH_NAME }}
+          git rm -rf *
+          mv ${{ github.workspace }}/pr_ui/* .
+          git add .
+          git commit -m "raylib screenshots for PR #${{ github.event.number }}"
+          git push origin ${{ env.BRANCH_NAME }} --force
+
+      - name: Comment Screenshots on PR
+        if: github.event_name == 'pull_request_target'
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          message: |
+            <!-- _(run_id_screenshots_raylib **${{ github.run_id }}**)_ -->
+            ## raylib UI Preview
+            ${{ steps.find_diff.outputs.DIFF }}
+          comment_tag: run_id_screenshots_raylib
+          pr_number: ${{ github.event.number }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/selfdrive_tests.yaml

@@ -277,3 +277,29 @@ jobs:
         with:
           name: report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
           path: selfdrive/ui/tests/test_ui/report_1/screenshots
+
+  create_raylib_ui_report:
+    name: Create raylib UI Report
+    runs-on: ${{
+      (github.repository == 'commaai/openpilot') &&
+      ((github.event_name != 'pull_request') ||
+      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
+      && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
+      || fromJSON('["ubuntu-24.04"]') }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: ./.github/workflows/setup-with-retry
+      - name: Build openpilot
+        run: ${{ env.RUN }} "scons -j$(nproc)"
+      - name: Create raylib UI Report
+        run: >
+          ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
+                          source selfdrive/test/setup_xvfb.sh &&
+                          python3 selfdrive/ui/tests/test_ui/raylib_screenshots.py"
+      - name: Upload Raylib UI Report
+        uses: actions/upload-artifact@v4
+        with:
+          name: raylib-report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
+          path: selfdrive/ui/tests/test_ui/raylib_report/screenshots

selfdrive/ui/layouts/home.py

@@ -56,6 +56,10 @@ class HomeLayout(Widget):
     self._exp_mode_button = ExperimentalModeButton()
     self._setup_callbacks()
 
+  def show_event(self):
+    self.last_refresh = time.monotonic()
+    self._refresh()
+
   def _setup_callbacks(self):
     self.update_alert.set_dismiss_callback(lambda: self._set_state(HomeLayoutState.HOME))
     self.offroad_alert.set_dismiss_callback(lambda: self._set_state(HomeLayoutState.HOME))

selfdrive/ui/tests/test_ui/raylib_screenshots.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+import os
+import sys
+import shutil
+import time
+import pathlib
+from collections import namedtuple
+
+import pyautogui
+import pywinctl
+
+from cereal import log
+from cereal import messaging
+from cereal.messaging import PubMaster
+from openpilot.common.params import Params
+from openpilot.common.prefix import OpenpilotPrefix
+from openpilot.selfdrive.test.helpers import with_processes
+from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert
+
+TEST_DIR = pathlib.Path(__file__).parent
+TEST_OUTPUT_DIR = TEST_DIR / "raylib_report"
+SCREENSHOTS_DIR = TEST_OUTPUT_DIR / "screenshots"
+UI_DELAY = 0.1
+
+# Offroad alerts to test
+OFFROAD_ALERTS = ['Offroad_IsTakingSnapshot']
+
+
+def setup_homescreen(click, pm: PubMaster):
+  pass
+
+
+def setup_settings_device(click, pm: PubMaster):
+  click(100, 100)
+
+
+def setup_settings_network(click, pm: PubMaster):
+  setup_settings_device(click, pm)
+  click(278, 450)
+
+
+def setup_settings_toggles(click, pm: PubMaster):
+  setup_settings_device(click, pm)
+  click(278, 600)
+
+
+def setup_settings_software(click, pm: PubMaster):
+  setup_settings_device(click, pm)
+  click(278, 720)
+
+
+def setup_settings_firehose(click, pm: PubMaster):
+  setup_settings_device(click, pm)
+  click(278, 845)
+
+
+def setup_settings_developer(click, pm: PubMaster):
+  setup_settings_device(click, pm)
+  click(278, 950)
+
+
+def setup_keyboard(click, pm: PubMaster):
+  setup_settings_developer(click, pm)
+  click(1930, 270)
+
+
+def setup_pair_device(click, pm: PubMaster):
+  click(1950, 800)
+
+
+def setup_offroad_alert(click, pm: PubMaster):
+  set_offroad_alert("Offroad_TemperatureTooHigh", True, extra_text='99')
+  for alert in OFFROAD_ALERTS:
+    set_offroad_alert(alert, True)
+
+  setup_settings_device(click, pm)
+  click(240, 216)
+
+
+CASES = {
+  "homescreen": setup_homescreen,
+  "settings_device": setup_settings_device,
+  "settings_network": setup_settings_network,
+  "settings_toggles": setup_settings_toggles,
+  "settings_software": setup_settings_software,
+  "settings_firehose": setup_settings_firehose,
+  "settings_developer": setup_settings_developer,
+  "keyboard": setup_keyboard,
+  "pair_device": setup_pair_device,
+  "offroad_alert": setup_offroad_alert,
+}
+
+
+class TestUI:
+  def __init__(self):
+    os.environ["SCALE"] = os.getenv("SCALE", "1")
+    sys.modules["mouseinfo"] = False
+
+  def setup(self):
+    # Seed minimal offroad state
+    self.pm = PubMaster(["deviceState"])
+    ds = messaging.new_message('deviceState')
+    ds.deviceState.networkType = log.DeviceState.NetworkType.wifi
+    for _ in range(5):
+      self.pm.send('deviceState', ds)
+      ds.clear_write_flag()
+      time.sleep(0.05)
+    time.sleep(0.5)
+    try:
+      self.ui = pywinctl.getWindowsWithTitle("UI")[0]
+    except Exception as e:
+      print(f"failed to find ui window, assuming that it's in the top left (for Xvfb) {e}")
+      self.ui = namedtuple("bb", ["left", "top", "width", "height"])(0, 0, 2160, 1080)
+
+  def screenshot(self, name: str):
+    full_screenshot = pyautogui.screenshot()
+    cropped = full_screenshot.crop((self.ui.left, self.ui.top, self.ui.left + self.ui.width, self.ui.top + self.ui.height))
+    cropped.save(SCREENSHOTS_DIR / f"{name}.png")
+
+  def click(self, x: int, y: int, *args, **kwargs):
+    pyautogui.mouseDown(self.ui.left + x, self.ui.top + y, *args, **kwargs)
+    time.sleep(0.01)
+    pyautogui.mouseUp(self.ui.left + x, self.ui.top + y, *args, **kwargs)
+
+  @with_processes(["raylib_ui"])
+  def test_ui(self, name, setup_case):
+    self.setup()
+    setup_case(self.click, self.pm)
+    self.screenshot(name)
+
+
+def create_screenshots():
+  if TEST_OUTPUT_DIR.exists():
+    shutil.rmtree(TEST_OUTPUT_DIR)
+  SCREENSHOTS_DIR.mkdir(parents=True)
+
+  t = TestUI()
+  with OpenpilotPrefix():
+    params = Params()
+    params.put("DongleId", "123456789012345")
+    for name, setup in CASES.items():
+      t.test_ui(name, setup)
+
+
+if __name__ == "__main__":
+  create_screenshots()