CI: generate test_ui report (#31151)

* add test

* simpler

* simpler

* false

* move these here

* faster

* map takes logner to render
old-commit-hash: faf99ba711

.github/workflows/selfdrive_tests.yaml

@@ -396,24 +396,24 @@ jobs:
               comment_id: ${{ steps.fc.outputs.comment-id }}
             })
 
-# need to figure out some stuff with tkinter before enabling this
+  create_ui_report:
-
+    name: Create UI Report
-  # create_ui_report:
+    runs-on: ubuntu-20.04
-  #   name: Create UI Report
+    steps:
-  #   runs-on: ubuntu-20.04
+      - uses: actions/checkout@v4
-  #   steps:
+        with:
-  #     - uses: actions/checkout@v4
+          submodules: true
-  #       with:
+      - uses: ./.github/workflows/setup-with-retry
-  #         submodules: true
+      - name: Build openpilot
-  #     - uses: ./.github/workflows/setup-with-retry
+        run: ${{ env.RUN }} "scons -j$(nproc)"
-  #     - name: Build openpilot
+      - name: Create Test Report
-  #       run: ${{ env.RUN }} "scons -j$(nproc)"
+        run: >
-  #     - name: Create Test Report
+            ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
-  #       run: ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
+                            source selfdrive/test/setup_xvfb.sh &&
-  #                            export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && \
+                            export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && 
-  #                            python selfdrive/ui/tests/test_ui/run.py"
+                            python selfdrive/ui/tests/test_ui/run.py"
-  #     - name: Upload Test Report
+      - name: Upload Test Report
-  #       uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v2
-  #       with:
+        with:
-  #         name: report
+          name: report
-  #         path: selfdrive/ui/tests/test_ui
+          path: selfdrive/ui/tests/test_ui/report

Dockerfile.openpilot_base

@@ -4,7 +4,7 @@ ENV PYTHONUNBUFFERED 1
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio && \
+    apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot && \
     rm -rf /var/lib/apt/lists/*
 
 RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen

selfdrive/navd/tests/test_map_renderer.py

@@ -28,8 +28,11 @@ LOCATION2_REPEATED = [LOCATION2] * DEFAULT_ITERATIONS
 def gen_llk(location=LOCATION1):
   msg = messaging.new_message('liveLocationKalman')
   msg.liveLocationKalman.positionGeodetic = {'value': [*location, 0], 'std': [0., 0., 0.], 'valid': True}
+  msg.liveLocationKalman.positionECEF = {'value': [0., 0., 0.], 'std': [0., 0., 0.], 'valid': True}
   msg.liveLocationKalman.calibratedOrientationNED = {'value': [0., 0., 0.], 'std': [0., 0., 0.], 'valid': True}
+  msg.liveLocationKalman.velocityCalibrated = {'value': [0., 0., 0.], 'std': [0., 0., 0.], 'valid': True}
   msg.liveLocationKalman.status = 'valid'
+  msg.liveLocationKalman.gpsOK = True
   return msg
 
 

selfdrive/test/setup_xvfb.sh

@@ -12,4 +12,8 @@ while [ ! -S /tmp/.X11-unix/X$DISP_ID ]
 do
   echo "Waiting for Xvfb..."
   sleep 1
-done
+done
+
+touch ~/.Xauthority
+export XDG_SESSION_TYPE="x11"
+xset -q

selfdrive/ui/tests/test_ui/run.py

@@ -1,10 +1,11 @@
+from collections import namedtuple
 import pathlib
 import shutil
+import sys
 import jinja2
 import matplotlib.pyplot as plt
 import numpy as np
 import os
-import pyautogui
 import pywinctl
 import time
 import unittest
@@ -17,6 +18,7 @@ from cereal.messaging import SubMaster, PubMaster
 from openpilot.common.params import Params
 from openpilot.common.realtime import DT_MDL
 from openpilot.common.transformations.camera import tici_f_frame_size
+from openpilot.selfdrive.navd.tests.test_map_renderer import gen_llk
 from openpilot.selfdrive.test.helpers import with_processes
 from openpilot.selfdrive.test.process_replay.vision_meta import meta_from_camera_state
 from openpilot.tools.webcam.camera import Camera
@@ -44,8 +46,6 @@ def setup_common(click, pm: PubMaster):
 
   pm.send("deviceState", dat)
 
-  time.sleep(UI_DELAY)
-
 def setup_homescreen(click, pm: PubMaster):
   setup_common(click, pm)
 
@@ -53,14 +53,12 @@ def setup_settings_device(click, pm: PubMaster):
   setup_common(click, pm)
 
   click(100, 100)
-  time.sleep(UI_DELAY)
 
 def setup_settings_network(click, pm: PubMaster):
   setup_common(click, pm)
 
   setup_settings_device(click, pm)
   click(300, 600)
-  time.sleep(UI_DELAY)
 
 def setup_onroad(click, pm: PubMaster):
   setup_common(click, pm)
@@ -77,7 +75,7 @@ def setup_onroad(click, pm: PubMaster):
   server.create_buffers(VisionStreamType.VISION_STREAM_WIDE_ROAD, 40, False, *tici_f_frame_size)
   server.start_listener()
 
-  time.sleep(UI_DELAY)
+  time.sleep(0.5) # give time for vipc server to start
 
   IMG = Camera.bgr2nv12(np.random.randint(0, 255, (*tici_f_frame_size,3), dtype=np.uint8))
   IMG_BYTES = IMG.flatten().tobytes()
@@ -96,17 +94,19 @@ def setup_onroad(click, pm: PubMaster):
     pm.send(msg.which(), msg)
     server.send(cam_meta.stream, IMG_BYTES, cs.frameId, cs.timestampSof, cs.timestampEof)
 
-  time.sleep(UI_DELAY)
-
 def setup_onroad_map(click, pm: PubMaster):
   setup_onroad(click, pm)
+
+  dat = gen_llk()
+  pm.send("liveLocationKalman", dat)
+
   click(500, 500)
-  time.sleep(UI_DELAY)
+
+  time.sleep(UI_DELAY) # give time for the map to render
 
 def setup_onroad_sidebar(click, pm: PubMaster):
   setup_onroad_map(click, pm)
   click(500, 500)
-  time.sleep(UI_DELAY)
 
 CASES = {
   "homescreen": setup_homescreen,
@@ -114,7 +114,7 @@ CASES = {
   "settings_network": setup_settings_network,
   "onroad": setup_onroad,
   "onroad_map": setup_onroad_map,
-  "onroad_map_sidebar": setup_onroad_sidebar
+  "onroad_sidebar": setup_onroad_sidebar
 }
 
 TEST_DIR = pathlib.Path(__file__).parent
@@ -127,16 +127,26 @@ class TestUI(unittest.TestCase):
   @classmethod
   def setUpClass(cls):
     os.environ["SCALE"] = "1"
+    sys.modules["mouseinfo"] = False
+
+  @classmethod
+  def tearDownClass(cls):
+    del sys.modules["mouseinfo"]
 
   def setup(self):
     self.sm = SubMaster(["uiDebug"])
-    self.pm = PubMaster(["deviceState", "pandaStates", "controlsState", 'roadCameraState', 'wideRoadCameraState'])
+    self.pm = PubMaster(["deviceState", "pandaStates", "controlsState", 'roadCameraState', 'wideRoadCameraState', 'liveLocationKalman'])
     while not self.sm.valid["uiDebug"]:
       self.sm.update(1)
-    time.sleep(UI_DELAY) # wait a bit more for the UI to finish rendering
+    time.sleep(UI_DELAY) # wait a bit more for the UI to start rendering
-    self.ui = pywinctl.getWindowsWithTitle("ui")[0]
+    try:
+      self.ui = pywinctl.getWindowsWithTitle("ui")[0]
+    except Exception as e:
+      print(f"failed to find ui window, assuming that it's in the top left (for Xvfb) {e}")
+      self.ui = namedtuple("bb", ["left", "top", "width", "height"])(0,0,2160,1080)
 
   def screenshot(self):
+    import pyautogui
     im = pyautogui.screenshot(region=(self.ui.left, self.ui.top, self.ui.width, self.ui.height))
     self.assertEqual(im.width, 2160)
     self.assertEqual(im.height, 1080)
@@ -145,7 +155,9 @@ class TestUI(unittest.TestCase):
     return img
 
   def click(self, x, y, *args, **kwargs):
+    import pyautogui
     pyautogui.click(self.ui.left + x, self.ui.top + y, *args, **kwargs)
+    time.sleep(UI_DELAY) # give enough time for the UI to react
 
   @parameterized.expand(CASES.items())
   @with_processes(["ui"])
@@ -154,6 +166,8 @@ class TestUI(unittest.TestCase):
 
     setup_case(self.click, self.pm)
 
+    time.sleep(UI_DELAY) # wait a bit more for the UI to finish rendering
+
     im = self.screenshot()
     plt.imsave(SCREENSHOTS_DIR / f"{name}.png", im)
 

tools/install_ubuntu_dependencies.sh

@@ -77,7 +77,6 @@ function install_ubuntu_common_requirements() {
     libqt5x11extras5-dev \
     libreadline-dev \
     libdw1 \
-    xvfb \
     valgrind
 }