ci: ui preview with diff (#33498)

* diff

* not for now

.github/workflows/selfdrive_tests.yaml

@@ -337,7 +337,6 @@ jobs:
     # This job name needs to be the same as UI_JOB_NAME in ui_preview.yaml
     name: Create UI Report
     runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request'
     steps:
       - uses: actions/checkout@v4
         with:
@@ -353,5 +352,5 @@ jobs:
       - name: Upload Test Report
         uses: actions/upload-artifact@v4
         with:
-          name: report-${{ github.event.number }}
+          name: report-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
           path: selfdrive/ui/tests/test_ui/report_1/screenshots

.github/workflows/ui_preview.yaml

@@ -1,5 +1,8 @@
 name: "ui preview"
 on:
+  push:
+    branches:
+      - master
   pull_request_target:
     types: [assigned, opened, synchronize, reopened, edited]
     branches:
@@ -10,6 +13,9 @@ on:
 
 env:
   UI_JOB_NAME: "Create UI Report"
+  REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
+  SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}
+  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}"
 
 jobs:
   preview:
@@ -22,91 +28,134 @@ jobs:
       pull-requests: write
       actions: read
     steps:
-      - name: Waiting for ui test to start
+      - name: Waiting for ui generation to start
         run: sleep 30
 
-      - name: Wait for ui report
+      - name: Waiting for ui generation to end
         uses: lewagon/wait-on-check-action@v1.3.4
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ env.SHA }}
           check-name: ${{ env.UI_JOB_NAME }}
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           allowed-conclusions: success
           wait-interval: 20
 
-      - name: Get workflow run ID
+      - name: Getting workflow run ID
         id: get_run_id
         run: |
-          echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ github.event.pull_request.head.sha }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
+          echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
 
-      - name: Checkout ci-artifacts
+      - name: Getting proposed ui
-        uses: actions/checkout@v4
-        with:
-          repository: commaai/ci-artifacts
-          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-          path: ${{ github.workspace }}/ci-artifacts
-          ref: master
-
-      - name: Download artifact
         id: download-artifact
         uses: dawidd6/action-download-artifact@v6
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           run_id: ${{ steps.get_run_id.outputs.run_id }}
           search_artifacts: true
-          name: report-${{ github.event.number }}
+          name: report-${{ env.REPORT_NAME }}
-          path: ${{ github.workspace }}/ci-artifacts
+          path: ${{ github.workspace }}/pr_ui
+
+      - name: Getting master ui
+        uses: actions/checkout@v4
+        with:
+          repository: commaai/ci-artifacts
+          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
+          path: ${{ github.workspace }}/master_ui
+          ref: openpilot_master_ui
+
+      - name: Saving new master ui
+        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+        working-directory: ${{ github.workspace }}/master_ui
+        run: |
+          git checkout --orphan=new_master_ui
+          git rm -rf *
+          git branch -D openpilot_master_ui
+          git branch -m openpilot_master_ui
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+          mv ${{ github.workspace }}/pr_ui/*.png .
+          git add .
+          git commit -m "screenshots for commit ${{ env.SHA }}"
+          git push origin openpilot_master_ui --force
+
+      - name: Finding diff
+        if: github.event_name == 'pull_request_target'
+        id: find_diff
+        run: >-
+          sudo apt-get install -y imagemagick
+
+          scenes="homescreen settings_device offroad_alert update_available prime onroad onroad_sidebar onroad_wide onroad_wide_sidebar onroad_alert_small onroad_alert_mid onroad_alert_full driver_camera body"
+          A=($scenes)
+
+          DIFF=""
+          TABLE="<details>"
+          TABLE="${TABLE}<summary>All Screenshots</summary>"
+          TABLE="${TABLE}<table>"
+
+          for ((i=0; i<${#A[*]}; i=i+1));
+          do
+
+            if ! compare -fuzz 2% -highlight-color DeepSkyBlue1 -lowlight-color Black -compose Src ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png; then
+              convert ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png -transparent black mask.png
+              composite mask.png ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png
+              convert -delay 20 ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png -loop 0 ${{ github.workspace }}/pr_ui/${A[$i]}_diff.gif
+
+              DIFF="${DIFF}<details>"
+              DIFF="${DIFF}<summary>${A[$i]} : \$\${\\color{red}\\text{DIFFERENT}}\$\$</summary>"
+              DIFF="${DIFF}<table>"
+
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> master <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot_master_ui/${A[$i]}.png\"> </td>"
+              DIFF="${DIFF}  <td> proposed <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+              DIFF="${DIFF}</tr>"
+
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> diff <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.png\"> </td>"
+              DIFF="${DIFF}  <td> composite diff <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.gif\"> </td>"
+              DIFF="${DIFF}</tr>"
+
+              DIFF="${DIFF}</table>"
+              DIFF="${DIFF}</details>"
+            else
+              rm -f ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png
+            fi
+
+            INDEX=$(($i % 2))
+            if [[ $INDEX -eq 0 ]]; then
+              TABLE="${TABLE}<tr>"
+            fi
+            TABLE="${TABLE}  <td> <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+            if [[ $INDEX -eq 1 || $(($i + 1)) -eq ${#A[*]} ]]; then
+              TABLE="${TABLE}</tr>"
+            fi
+          done
+
+          TABLE="${TABLE}</table>"
+          TABLE="${TABLE}</details>"
+
+          echo "DIFF=$DIFF$TABLE" >> "$GITHUB_OUTPUT"
 
-      - name: Push Screenshots
+      - name: Saving proposed ui
-        working-directory: ${{ github.workspace }}/ci-artifacts
+        if: github.event_name == 'pull_request_target'
+        working-directory: ${{ github.workspace }}/master_ui
         run: |
-          git checkout -b openpilot/pr-${{ github.event.number }}
           git config user.name "GitHub Actions Bot"
           git config user.email "<>"
-          git add ${{ github.workspace }}/ci-artifacts/*
+          git checkout --orphan=${{ env.BRANCH_NAME }}
+          git rm -rf *
+          mv ${{ github.workspace }}/pr_ui/* .
+          git add .
           git commit -m "screenshots for PR #${{ github.event.number }}"
-          git push origin openpilot/pr-${{ github.event.number }} --force
+          git push origin ${{ env.BRANCH_NAME }} --force
 
       - name: Comment Screenshots on PR
+        if: github.event_name == 'pull_request_target'
         uses: thollander/actions-comment-pull-request@v2
         with:
           message: |
             <!-- _(run_id_screenshots **${{ github.run_id }}**)_ -->
-            ## UI Screenshots
+            ## UI Preview
-            <table>
+            ${{ steps.find_diff.outputs.DIFF }}
-              <tr>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/homescreen.png"></td>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/settings_device.png"></td>
-              </tr>
-              <tr>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/offroad_alert.png"></td>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/update_available.png"></td>
-              </tr>
-              <tr>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/prime.png"></td>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/pair_device.png"></td>
-              </tr>
-              <tr>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/onroad.png"></td>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/onroad_sidebar.png"></td>
-              </tr>
-              <tr>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/onroad_wide.png"></td>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/onroad_wide_sidebar.png"></td>
-              </tr>
-              <tr>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/onroad_alert_small.png"></td>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/onroad_alert_mid.png"></td>
-              </tr>
-              <tr>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/onroad_alert_full.png"></td>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/driver_camera.png"></td>
-              </tr>
-              <tr>
-                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/body.png"></td>
-                <td></td>
-              </tr>
-            </table>
           comment_tag: run_id_screenshots
           pr_number: ${{ github.event.number }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}