diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index e98b4a19f7..b510e8509d 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -352,19 +352,29 @@ jobs: create_ui_report: # This job name needs to be the same as UI_JOB_NAME in ui_preview.yaml name: Create UI Report - runs-on: ubuntu-latest + runs-on: ${{ ((github.repository == 'commaai/openpilot') && + ((github.event_name != 'pull_request') || + (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }} steps: - uses: actions/checkout@v4 with: submodules: true - uses: ./.github/workflows/setup-with-retry + - name: caching frames + id: frames-cache + uses: actions/cache@v4 + with: + path: .ci_cache/comma_download_cache + key: ui_screenshots_test_${{ hashFiles('selfdrive/ui/tests/test_ui/run.py') }} - name: Build openpilot run: ${{ env.RUN }} "scons -j$(nproc)" - name: Create Test Report + timeout-minutes: ${{ ((steps.frames-cache.outputs.cache-hit == 'true') && 1 || 3) }} run: > ${{ env.RUN }} "PYTHONWARNINGS=ignore && source selfdrive/test/setup_xvfb.sh && - python3 selfdrive/ui/tests/test_ui/run.py" + CACHE_ROOT=/tmp/comma_download_cache python3 selfdrive/ui/tests/test_ui/run.py && + chmod -R 777 /tmp/comma_download_cache" - name: Upload Test Report uses: actions/upload-artifact@v4 with: diff --git a/selfdrive/ui/tests/test_ui/run.py b/selfdrive/ui/tests/test_ui/run.py index e6fd9a016a..8cd5f1930d 100644 --- a/selfdrive/ui/tests/test_ui/run.py +++ b/selfdrive/ui/tests/test_ui/run.py @@ -3,11 +3,10 @@ import capnp import pathlib import shutil import sys -import jinja2 -import matplotlib.pyplot as plt -import numpy as np import os import pywinctl +import pyautogui +import pickle import time from cereal import log @@ -23,8 +22,9 @@ from openpilot.selfdrive.test.process_replay.migration import migrate, migrate_c from openpilot.tools.lib.logreader import LogReader from openpilot.tools.lib.framereader import FrameReader from openpilot.tools.lib.route import Route +from openpilot.tools.lib.cache import DEFAULT_CACHE_DIR -UI_DELAY = 0.5 # may be slower on CI? +UI_DELAY = 0.1 # may be slower on CI? TEST_ROUTE = "a2a0ccea32023010|2023-07-27--13-01-19" STREAMS: list[tuple[VisionStreamType, CameraConfig, bytes]] = [] @@ -222,41 +222,20 @@ class TestUI: print(f"failed to find ui window, assuming that it's in the top left (for Xvfb) {e}") self.ui = namedtuple("bb", ["left", "top", "width", "height"])(0,0,2160,1080) - def screenshot(self): - import pyautogui - im = pyautogui.screenshot(region=(self.ui.left, self.ui.top, self.ui.width, self.ui.height)) + def screenshot(self, name): + im = pyautogui.screenshot(SCREENSHOTS_DIR / f"{name}.png", region=(self.ui.left, self.ui.top, self.ui.width, self.ui.height)) assert im.width == 2160 assert im.height == 1080 - img = np.array(im) - im.close() - return img def click(self, x, y, *args, **kwargs): - import pyautogui pyautogui.click(self.ui.left + x, self.ui.top + y, *args, **kwargs) time.sleep(UI_DELAY) # give enough time for the UI to react @with_processes(["ui"]) def test_ui(self, name, setup_case): self.setup() - setup_case(self.click, self.pm) - - im = self.screenshot() - plt.imsave(SCREENSHOTS_DIR / f"{name}.png", im) - - -def create_html_report(): - OUTPUT_FILE = TEST_OUTPUT_DIR / "index.html" - - with open(TEST_DIR / "template.html") as f: - template = jinja2.Template(f.read()) - - cases = {f.stem: (str(f.relative_to(TEST_OUTPUT_DIR)), "reference.png") for f in SCREENSHOTS_DIR.glob("*.png")} - cases = dict(sorted(cases.items())) - - with open(OUTPUT_FILE, "w") as f: - f.write(template.render(cases=cases)) + self.screenshot(name) def create_screenshots(): if TEST_OUTPUT_DIR.exists(): @@ -277,13 +256,23 @@ def create_screenshots(): break cam = DEVICE_CAMERAS[("tici", "ar0231")] - road_img = FrameReader(route.camera_paths()[segnum]).get(0, pix_fmt="nv12")[0] - STREAMS.append((VisionStreamType.VISION_STREAM_ROAD, cam.fcam, road_img.flatten().tobytes())) - wide_road_img = FrameReader(route.ecamera_paths()[segnum]).get(0, pix_fmt="nv12")[0] - STREAMS.append((VisionStreamType.VISION_STREAM_WIDE_ROAD, cam.ecam, wide_road_img.flatten().tobytes())) + frames_cache = f'{DEFAULT_CACHE_DIR}/ui_frames' + if os.path.isfile(frames_cache): + with open(frames_cache, 'rb') as f: + frames = pickle.load(f) + road_img = frames[0] + wide_road_img = frames[1] + driver_img = frames[2] + else: + with open(frames_cache, 'wb') as f: + road_img = FrameReader(route.camera_paths()[segnum]).get(0, pix_fmt="nv12")[0] + wide_road_img = FrameReader(route.ecamera_paths()[segnum]).get(0, pix_fmt="nv12")[0] + driver_img = FrameReader(route.dcamera_paths()[segnum]).get(0, pix_fmt="nv12")[0] + pickle.dump([road_img, wide_road_img, driver_img], f) - driver_img = FrameReader(route.dcamera_paths()[segnum]).get(0, pix_fmt="nv12")[0] + STREAMS.append((VisionStreamType.VISION_STREAM_ROAD, cam.fcam, road_img.flatten().tobytes())) + STREAMS.append((VisionStreamType.VISION_STREAM_WIDE_ROAD, cam.ecam, wide_road_img.flatten().tobytes())) STREAMS.append((VisionStreamType.VISION_STREAM_DRIVER, cam.dcam, driver_img.flatten().tobytes())) t = TestUI() @@ -302,6 +291,3 @@ def create_screenshots(): if __name__ == "__main__": print("creating test screenshots") create_screenshots() - - print("creating html report") - create_html_report()