From ef93981bfa3116442729ffe1b98a7c6f817c68ad Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Sat, 27 Sep 2025 02:37:35 -0700 Subject: [PATCH] raylib: ui diff test (#36213) * add raylib ui test * match qt * exe * vibing is epic * this is epic * format * add more settings * fix to actually use raylib * add kb * global * pair * rm cmts * show event * this is so stupid clean up * clean up * rename dir * clean up * no more vibe * rm * ugh it's always slightly different for no reason * nvm region is actually broken * 1l --- .github/workflows/raylib_ui_preview.yaml | 174 ++++++++++++++++++ .github/workflows/selfdrive_tests.yaml | 26 +++ selfdrive/ui/layouts/home.py | 4 + .../ui/tests/test_ui/raylib_screenshots.py | 146 +++++++++++++++ 4 files changed, 350 insertions(+) create mode 100644 .github/workflows/raylib_ui_preview.yaml create mode 100755 selfdrive/ui/tests/test_ui/raylib_screenshots.py diff --git a/.github/workflows/raylib_ui_preview.yaml b/.github/workflows/raylib_ui_preview.yaml new file mode 100644 index 0000000000..11e745a96a --- /dev/null +++ b/.github/workflows/raylib_ui_preview.yaml @@ -0,0 +1,174 @@ +name: "raylib ui preview" +on: + push: + branches: + - master + pull_request_target: + types: [assigned, opened, synchronize, reopened, edited] + branches: + - 'master' + paths: + - 'selfdrive/ui/**' + - 'system/ui/**' + workflow_dispatch: + +env: + UI_JOB_NAME: "Create raylib UI Report" + REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }} + SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }} + BRANCH_NAME: "openpilot/pr-${{ github.event.number }}-raylib-ui" + +jobs: + preview: + if: github.repository == 'commaai/openpilot' + name: preview + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + pull-requests: write + actions: read + steps: + - name: Waiting for ui generation to start + run: sleep 30 + + - name: Waiting for ui generation to end + uses: lewagon/wait-on-check-action@v1.3.4 + with: + ref: ${{ env.SHA }} + check-name: ${{ env.UI_JOB_NAME }} + repo-token: ${{ secrets.GITHUB_TOKEN }} + allowed-conclusions: success + wait-interval: 20 + + - name: Getting workflow run ID + id: get_run_id + run: | + echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?[0-9]+)") | .number')" >> $GITHUB_OUTPUT + + - name: Getting proposed ui + id: download-artifact + uses: dawidd6/action-download-artifact@v6 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + run_id: ${{ steps.get_run_id.outputs.run_id }} + search_artifacts: true + name: raylib-report-${{ env.REPORT_NAME }} + path: ${{ github.workspace }}/pr_ui + + - name: Getting master ui + uses: actions/checkout@v4 + with: + repository: commaai/ci-artifacts + ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }} + path: ${{ github.workspace }}/master_ui_raylib + ref: openpilot_master_ui_raylib + + - name: Saving new master ui + if: github.ref == 'refs/heads/master' && github.event_name == 'push' + working-directory: ${{ github.workspace }}/master_ui_raylib + run: | + git checkout --orphan=new_master_ui_raylib + git rm -rf * + git branch -D openpilot_master_ui_raylib + git branch -m openpilot_master_ui_raylib + git config user.name "GitHub Actions Bot" + git config user.email "<>" + mv ${{ github.workspace }}/pr_ui/*.png . + git add . + git commit -m "raylib screenshots for commit ${{ env.SHA }}" + git push origin openpilot_master_ui_raylib --force + + - name: Finding diff + if: github.event_name == 'pull_request_target' + id: find_diff + run: >- + sudo apt-get update && sudo apt-get install -y imagemagick + + scenes=$(find ${{ github.workspace }}/pr_ui/*.png -type f -printf "%f\n" | cut -d '.' -f 1 | grep -v 'pair_device') + A=($scenes) + + DIFF="" + TABLE="
All Screenshots" + TABLE="${TABLE}" + + for ((i=0; i<${#A[*]}; i=i+1)); + do + # Check if the master file exists + if [ ! -f "${{ github.workspace }}/master_ui_raylib/${A[$i]}.png" ]; then + # This is a new file in PR UI that doesn't exist in master + DIFF="${DIFF}
" + DIFF="${DIFF}${A[$i]} : \$\${\\color{cyan}\\text{NEW}}\$\$" + DIFF="${DIFF}
" + + DIFF="${DIFF}" + DIFF="${DIFF} " + DIFF="${DIFF}" + + DIFF="${DIFF}
" + DIFF="${DIFF}
" + elif ! compare -fuzz 2% -highlight-color DeepSkyBlue1 -lowlight-color Black -compose Src ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png; then + convert ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png -transparent black mask.png + composite mask.png ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png composite_diff.png + convert -delay 100 ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png composite_diff.png -loop 0 ${{ github.workspace }}/pr_ui/${A[$i]}_diff.gif + + mv ${{ github.workspace }}/master_ui_raylib/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_master_ref.png + + DIFF="${DIFF}
" + DIFF="${DIFF}${A[$i]} : \$\${\\color{red}\\text{DIFFERENT}}\$\$" + DIFF="${DIFF}" + + DIFF="${DIFF}" + DIFF="${DIFF} " + DIFF="${DIFF} " + DIFF="${DIFF}" + + DIFF="${DIFF}" + DIFF="${DIFF} " + DIFF="${DIFF} " + DIFF="${DIFF}" + + DIFF="${DIFF}
master proposed
diff composite diff
" + DIFF="${DIFF}
" + else + rm -f ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png + fi + + INDEX=$(($i % 2)) + if [[ $INDEX -eq 0 ]]; then + TABLE="${TABLE}" + fi + TABLE="${TABLE} " + if [[ $INDEX -eq 1 || $(($i + 1)) -eq ${#A[*]} ]]; then + TABLE="${TABLE}" + fi + done + + TABLE="${TABLE}" + + echo "DIFF=$DIFF$TABLE" >> "$GITHUB_OUTPUT" + + - name: Saving proposed ui + if: github.event_name == 'pull_request_target' + working-directory: ${{ github.workspace }}/master_ui_raylib + run: | + git config user.name "GitHub Actions Bot" + git config user.email "<>" + git checkout --orphan=${{ env.BRANCH_NAME }} + git rm -rf * + mv ${{ github.workspace }}/pr_ui/* . + git add . + git commit -m "raylib screenshots for PR #${{ github.event.number }}" + git push origin ${{ env.BRANCH_NAME }} --force + + - name: Comment Screenshots on PR + if: github.event_name == 'pull_request_target' + uses: thollander/actions-comment-pull-request@v2 + with: + message: | + + ## raylib UI Preview + ${{ steps.find_diff.outputs.DIFF }} + comment_tag: run_id_screenshots_raylib + pr_number: ${{ github.event.number }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index beb426c669..57b1158be2 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -277,3 +277,29 @@ jobs: with: name: report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }} path: selfdrive/ui/tests/test_ui/report_1/screenshots + + create_raylib_ui_report: + name: Create raylib UI Report + runs-on: ${{ + (github.repository == 'commaai/openpilot') && + ((github.event_name != 'pull_request') || + (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) + && fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]') + || fromJSON('["ubuntu-24.04"]') }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: ./.github/workflows/setup-with-retry + - name: Build openpilot + run: ${{ env.RUN }} "scons -j$(nproc)" + - name: Create raylib UI Report + run: > + ${{ env.RUN }} "PYTHONWARNINGS=ignore && + source selfdrive/test/setup_xvfb.sh && + python3 selfdrive/ui/tests/test_ui/raylib_screenshots.py" + - name: Upload Raylib UI Report + uses: actions/upload-artifact@v4 + with: + name: raylib-report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }} + path: selfdrive/ui/tests/test_ui/raylib_report/screenshots diff --git a/selfdrive/ui/layouts/home.py b/selfdrive/ui/layouts/home.py index 9ee533d2a4..320e7477f1 100644 --- a/selfdrive/ui/layouts/home.py +++ b/selfdrive/ui/layouts/home.py @@ -56,6 +56,10 @@ class HomeLayout(Widget): self._exp_mode_button = ExperimentalModeButton() self._setup_callbacks() + def show_event(self): + self.last_refresh = time.monotonic() + self._refresh() + def _setup_callbacks(self): self.update_alert.set_dismiss_callback(lambda: self._set_state(HomeLayoutState.HOME)) self.offroad_alert.set_dismiss_callback(lambda: self._set_state(HomeLayoutState.HOME)) diff --git a/selfdrive/ui/tests/test_ui/raylib_screenshots.py b/selfdrive/ui/tests/test_ui/raylib_screenshots.py new file mode 100755 index 0000000000..7fb22e4484 --- /dev/null +++ b/selfdrive/ui/tests/test_ui/raylib_screenshots.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +import os +import sys +import shutil +import time +import pathlib +from collections import namedtuple + +import pyautogui +import pywinctl + +from cereal import log +from cereal import messaging +from cereal.messaging import PubMaster +from openpilot.common.params import Params +from openpilot.common.prefix import OpenpilotPrefix +from openpilot.selfdrive.test.helpers import with_processes +from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert + +TEST_DIR = pathlib.Path(__file__).parent +TEST_OUTPUT_DIR = TEST_DIR / "raylib_report" +SCREENSHOTS_DIR = TEST_OUTPUT_DIR / "screenshots" +UI_DELAY = 0.1 + +# Offroad alerts to test +OFFROAD_ALERTS = ['Offroad_IsTakingSnapshot'] + + +def setup_homescreen(click, pm: PubMaster): + pass + + +def setup_settings_device(click, pm: PubMaster): + click(100, 100) + + +def setup_settings_network(click, pm: PubMaster): + setup_settings_device(click, pm) + click(278, 450) + + +def setup_settings_toggles(click, pm: PubMaster): + setup_settings_device(click, pm) + click(278, 600) + + +def setup_settings_software(click, pm: PubMaster): + setup_settings_device(click, pm) + click(278, 720) + + +def setup_settings_firehose(click, pm: PubMaster): + setup_settings_device(click, pm) + click(278, 845) + + +def setup_settings_developer(click, pm: PubMaster): + setup_settings_device(click, pm) + click(278, 950) + + +def setup_keyboard(click, pm: PubMaster): + setup_settings_developer(click, pm) + click(1930, 270) + + +def setup_pair_device(click, pm: PubMaster): + click(1950, 800) + + +def setup_offroad_alert(click, pm: PubMaster): + set_offroad_alert("Offroad_TemperatureTooHigh", True, extra_text='99') + for alert in OFFROAD_ALERTS: + set_offroad_alert(alert, True) + + setup_settings_device(click, pm) + click(240, 216) + + +CASES = { + "homescreen": setup_homescreen, + "settings_device": setup_settings_device, + "settings_network": setup_settings_network, + "settings_toggles": setup_settings_toggles, + "settings_software": setup_settings_software, + "settings_firehose": setup_settings_firehose, + "settings_developer": setup_settings_developer, + "keyboard": setup_keyboard, + "pair_device": setup_pair_device, + "offroad_alert": setup_offroad_alert, +} + + +class TestUI: + def __init__(self): + os.environ["SCALE"] = os.getenv("SCALE", "1") + sys.modules["mouseinfo"] = False + + def setup(self): + # Seed minimal offroad state + self.pm = PubMaster(["deviceState"]) + ds = messaging.new_message('deviceState') + ds.deviceState.networkType = log.DeviceState.NetworkType.wifi + for _ in range(5): + self.pm.send('deviceState', ds) + ds.clear_write_flag() + time.sleep(0.05) + time.sleep(0.5) + try: + self.ui = pywinctl.getWindowsWithTitle("UI")[0] + except Exception as e: + print(f"failed to find ui window, assuming that it's in the top left (for Xvfb) {e}") + self.ui = namedtuple("bb", ["left", "top", "width", "height"])(0, 0, 2160, 1080) + + def screenshot(self, name: str): + full_screenshot = pyautogui.screenshot() + cropped = full_screenshot.crop((self.ui.left, self.ui.top, self.ui.left + self.ui.width, self.ui.top + self.ui.height)) + cropped.save(SCREENSHOTS_DIR / f"{name}.png") + + def click(self, x: int, y: int, *args, **kwargs): + pyautogui.mouseDown(self.ui.left + x, self.ui.top + y, *args, **kwargs) + time.sleep(0.01) + pyautogui.mouseUp(self.ui.left + x, self.ui.top + y, *args, **kwargs) + + @with_processes(["raylib_ui"]) + def test_ui(self, name, setup_case): + self.setup() + setup_case(self.click, self.pm) + self.screenshot(name) + + +def create_screenshots(): + if TEST_OUTPUT_DIR.exists(): + shutil.rmtree(TEST_OUTPUT_DIR) + SCREENSHOTS_DIR.mkdir(parents=True) + + t = TestUI() + with OpenpilotPrefix(): + params = Params() + params.put("DongleId", "123456789012345") + for name, setup in CASES.items(): + t.test_ui(name, setup) + + +if __name__ == "__main__": + create_screenshots()