test_ui: create test ui screenshots and html report (#31147)

* test ui

* report + common

* in ci

* fix

* dont enable in ci yet
old-commit-hash: e2ec5be6ee

.github/workflows/selfdrive_tests.yaml

@@ -395,3 +395,25 @@ jobs:
               repo: context.repo.repo,
               comment_id: ${{ steps.fc.outputs.comment-id }}
             })
+
+# need to figure out some stuff with tkinter before enabling this
+
+  # create_ui_report:
+  #   name: Create UI Report
+  #   runs-on: ubuntu-20.04
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         submodules: true
+  #     - uses: ./.github/workflows/setup-with-retry
+  #     - name: Build openpilot
+  #       run: ${{ env.RUN }} "scons -j$(nproc)"
+  #     - name: Create Test Report
+  #       run: ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
+  #                            export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && \
+  #                            python selfdrive/ui/tests/test_ui/run.py"
+  #     - name: Upload Test Report
+  #       uses: actions/upload-artifact@v2
+  #       with:
+  #         name: report
+  #         path: selfdrive/ui/tests/test_ui

poetry.lock

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff48c093d927ffdfd6a721dfc172ccb8379b686ee66266613dc60b8bc2bcdf46
+oid sha256:06e996a4a287ab1fd52e1dc48696be65b8e61826e0b2c497e4efc44ae62b5e2c
-size 432802
+size 586723

pyproject.toml

@@ -136,8 +136,10 @@ parameterized = "^0.8"
 pprofile = "*"
 polyline = "*"
 pre-commit = "*"
+pyautogui = "*"
 pyopencl = "*"
 pygame = "*"
+pywinctl = "*"
 pyprof2calltree = "*"
 pytest = "*"
 pytest-cov = "*"

selfdrive/test/setup_xvfb.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Sets up a virtual display for running map renderer and simulator without an X11 display
 

selfdrive/ui/tests/.gitignore

@@ -3,3 +3,4 @@ playsound
 test_sound
 test_translations
 ui_snapshot
+test_ui/report

selfdrive/ui/tests/test_ui/run.py

@@ -0,0 +1,121 @@
+import pathlib
+import shutil
+import jinja2
+import matplotlib.pyplot as plt
+import numpy as np
+import os
+import pyautogui
+import pywinctl
+import time
+import unittest
+
+from parameterized import parameterized
+from cereal import messaging, log
+
+from cereal.messaging import SubMaster, PubMaster
+from openpilot.common.params import Params
+from openpilot.selfdrive.test.helpers import with_processes
+
+UI_DELAY = 0.5 # may be slower on CI?
+
+NetworkType = log.DeviceState.NetworkType
+NetworkStrength = log.DeviceState.NetworkStrength
+
+def setup_common(click, pm: PubMaster):
+  Params().put("DongleId", "123456789012345")
+  dat = messaging.new_message('deviceState')
+  dat.deviceState.networkType = NetworkType.cell4G
+  dat.deviceState.networkStrength = NetworkStrength.moderate
+
+  pm.send("deviceState", dat)
+
+  time.sleep(UI_DELAY)
+
+def setup_homescreen(click, pm: PubMaster):
+  setup_common(click, pm)
+
+def setup_settings_device(click, pm: PubMaster):
+  setup_common(click, pm)
+
+  click(100, 100)
+  time.sleep(UI_DELAY)
+
+def setup_settings_network(click, pm: PubMaster):
+  setup_common(click, pm)
+
+  setup_settings_device(click, pm)
+  click(300, 600)
+  time.sleep(UI_DELAY)
+
+CASES = {
+  "homescreen": setup_homescreen,
+  "settings_device": setup_settings_device,
+  "settings_network": setup_settings_network,
+}
+
+
+TEST_DIR = pathlib.Path(__file__).parent
+
+TEST_OUTPUT_DIR = TEST_DIR / "report"
+SCREENSHOTS_DIR = TEST_OUTPUT_DIR / "screenshots"
+
+
+class TestUI(unittest.TestCase):
+  @classmethod
+  def setUpClass(cls):
+    os.environ["SCALE"] = "1"
+
+  def setup(self):
+    self.sm = SubMaster(["uiDebug"])
+    self.pm = PubMaster(["deviceState"])
+    while not self.sm.valid["uiDebug"]:
+      self.sm.update(1)
+    time.sleep(UI_DELAY) # wait a bit more for the UI to finish rendering
+    self.ui = pywinctl.getWindowsWithTitle("ui")[0]
+
+  def screenshot(self):
+    im = pyautogui.screenshot(region=(self.ui.left, self.ui.top, self.ui.width, self.ui.height))
+    self.assertEqual(im.width, 2160)
+    self.assertEqual(im.height, 1080)
+    img = np.array(im)
+    im.close()
+    return img
+
+  def click(self, x, y, *args, **kwargs):
+    pyautogui.click(self.ui.left + x, self.ui.top + y, *args, **kwargs)
+
+  @parameterized.expand(CASES.items())
+  @with_processes(["ui"])
+  def test_ui(self, name, setup_case):
+    self.setup()
+
+    setup_case(self.click, self.pm)
+
+    im = self.screenshot()
+    plt.imsave(SCREENSHOTS_DIR / f"{name}.png", im)
+
+
+def create_html_report():
+  OUTPUT_FILE = TEST_OUTPUT_DIR / "index.html"
+
+  with open(TEST_DIR / "template.html") as f:
+    template = jinja2.Template(f.read())
+
+  cases = {f.stem: (str(f.relative_to(TEST_OUTPUT_DIR)), "reference.png") for f in SCREENSHOTS_DIR.glob("*.png")}
+
+  with open(OUTPUT_FILE, "w") as f:
+    f.write(template.render(cases=cases))
+
+def create_screenshots():
+  if TEST_OUTPUT_DIR.exists():
+    shutil.rmtree(TEST_OUTPUT_DIR)
+
+  SCREENSHOTS_DIR.mkdir(parents=True)
+  unittest.main(exit=False)
+
+if __name__ == "__main__":
+  print("creating test screenshots")
+  create_screenshots()
+
+  print("creating html report")
+  create_html_report()

selfdrive/ui/tests/test_ui/template.html

@@ -0,0 +1,34 @@
+<html>
+
+<style>
+ .column {
+  float: left;
+  width: 50%;
+  padding: 5px;
+}
+
+.row::after {
+  content: "";
+  clear: both;
+  display: table;
+} 
+
+.image {
+  width: 100%;
+}
+
+</style>
+
+{% for name, (image, ref_image) in cases.items() %}
+
+<h1>{{name}}</h1>
+<div class="row">
+  <div class="column">
+    <img class="image" src="{{ image }}"  />
+  </div>
+</div>
+
+<br>
+
+{% endfor %}
+</html>