diff --git a/.github/workflows/badges.yaml b/.github/workflows/badges.yaml index c19be4c563..923e249849 100644 --- a/.github/workflows/badges.yaml +++ b/.github/workflows/badges.yaml @@ -7,7 +7,7 @@ on: env: BASE_IMAGE: openpilot-base DOCKER_REGISTRY: ghcr.io/commaai - RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/sh -c + RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c jobs: badges: diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index c1d4eae52f..6ad273a14d 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -15,7 +15,7 @@ env: BUILD: selfdrive/test/docker_build.sh base - RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c + RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c jobs: docs: diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index c47947f90d..708f16368f 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -20,11 +20,11 @@ env: DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} BUILD: selfdrive/test/docker_build.sh base - RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c + RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c BUILD_CL: selfdrive/test/docker_build.sh cl - RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c + RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/bash -c PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 @@ -175,7 +175,9 @@ jobs: - name: Run unit tests timeout-minutes: 15 run: | - ${{ env.RUN }} "$PYTEST --timeout 30 -m 'not slow' && \ + ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \ + export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && \ + $PYTEST --timeout 30 -m 'not slow' && \ ./selfdrive/ui/tests/create_test_translations.sh && \ QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \ ./selfdrive/ui/tests/test_translations.py && \ diff --git a/.github/workflows/tools_tests.yaml b/.github/workflows/tools_tests.yaml index 75a34b3c55..d564606c3e 100644 --- a/.github/workflows/tools_tests.yaml +++ b/.github/workflows/tools_tests.yaml @@ -17,11 +17,11 @@ env: BUILD: selfdrive/test/docker_build.sh base - RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c + RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c BUILD_CL: selfdrive/test/docker_build.sh cl - RUN_CL: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c + RUN_CL: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/bash -c jobs: diff --git a/Jenkinsfile b/Jenkinsfile index 2118c4c4ed..926a9bbfc8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -187,7 +187,6 @@ node { ["test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py"], ["test pigeond", "pytest system/sensord/tests/test_pigeond.py"], ["test manager", "pytest selfdrive/manager/test/test_manager.py"], - ["test nav", "pytest selfdrive/navd/tests/"], ]) }, 'loopback': { diff --git a/pyproject.toml b/pyproject.toml index 58c506e378..07cf9a2233 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ testpaths = [ "selfdrive/controls", "selfdrive/locationd", "selfdrive/monitoring", + "selfdrive/navd/tests", "selfdrive/thermald", "selfdrive/test/longitudinal_maneuvers", "system/hardware/tici", diff --git a/selfdrive/navd/tests/test_map_renderer.py b/selfdrive/navd/tests/test_map_renderer.py index dbd6add995..d336ffb651 100755 --- a/selfdrive/navd/tests/test_map_renderer.py +++ b/selfdrive/navd/tests/test_map_renderer.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import time import numpy as np import os import pytest @@ -10,7 +11,7 @@ import cereal.messaging as messaging from typing import Any from cereal.visionipc import VisionIpcClient, VisionStreamType -from openpilot.selfdrive.manager.process_config import managed_processes +from openpilot.selfdrive.test.helpers import with_processes LLK_DECIMATION = 10 CACHE_PATH = "/data/mbgl-cache-navd.db" @@ -18,7 +19,8 @@ CACHE_PATH = "/data/mbgl-cache-navd.db" LOCATION1 = (32.7174, -117.16277) LOCATION2 = (32.7558, -117.2037) -DEFAULT_ITERATIONS = 30 * LLK_DECIMATION +RENDER_FRAMES = 15 +DEFAULT_ITERATIONS = RENDER_FRAMES * LLK_DECIMATION LOCATION1_REPEATED = [LOCATION1] * DEFAULT_ITERATIONS LOCATION2_REPEATED = [LOCATION2] * DEFAULT_ITERATIONS @@ -60,7 +62,8 @@ class MapBoxInternetDisabledRequestHandler(http.server.BaseHTTPRequestHandler): class MapBoxInternetDisabledServer(threading.Thread): def run(self): - self.server = http.server.HTTPServer(("127.0.0.1", 5000), MapBoxInternetDisabledRequestHandler) + self.server = http.server.HTTPServer(("127.0.0.1", 0), MapBoxInternetDisabledRequestHandler) + self.port = self.server.server_port self.server.serve_forever() def stop(self): @@ -74,13 +77,15 @@ class MapBoxInternetDisabledServer(threading.Thread): class TestMapRenderer(unittest.TestCase): - server = MapBoxInternetDisabledServer() + server: MapBoxInternetDisabledServer @classmethod def setUpClass(cls): assert "MAPBOX_TOKEN" in os.environ cls.original_token = os.environ["MAPBOX_TOKEN"] + cls.server = MapBoxInternetDisabledServer() cls.server.start() + time.sleep(0.5) # wait for server to startup @classmethod def tearDownClass(cls) -> None: @@ -88,7 +93,7 @@ class TestMapRenderer(unittest.TestCase): def setUp(self): self.server.enable_internet() - os.environ['MAPS_HOST'] = 'http://localhost:5000' + os.environ['MAPS_HOST'] = f'http://localhost:{self.server.port}' self.sm = messaging.SubMaster(['mapRenderState']) self.pm = messaging.PubMaster(['liveLocationKalman']) @@ -97,15 +102,11 @@ class TestMapRenderer(unittest.TestCase): if os.path.exists(CACHE_PATH): os.remove(CACHE_PATH) - def tearDown(self): - managed_processes['mapsd'].stop() - def _setup_test(self): - # start + sync up - managed_processes['mapsd'].start() - assert self.pm.wait_for_readers_to_update("liveLocationKalman", 10) + time.sleep(0.5) + assert VisionIpcClient.available_streams("navd", False) == {VisionStreamType.VISION_STREAM_MAP, } assert self.vipc.connect(False) self.vipc.recv() @@ -164,17 +165,22 @@ class TestMapRenderer(unittest.TestCase): assert self.vipc.timestamp_sof == llk.logMonoTime assert self.vipc.frame_id == self.sm['mapRenderState'].frameId + assert frames_since_test_start >= RENDER_FRAMES + return render_times + @with_processes(["mapsd"]) def test_with_internet(self): self._setup_test() self._run_test(True) + @with_processes(["mapsd"]) def test_with_no_internet(self): self.server.disable_internet() self._setup_test() self._run_test(False) + @with_processes(["mapsd"]) def test_recover_from_no_internet(self): self._setup_test() self._run_test(True) @@ -187,8 +193,7 @@ class TestMapRenderer(unittest.TestCase): self.server.enable_internet() self._run_test(True, LOCATION2_REPEATED) - self._run_test(True, LOCATION2_REPEATED) - + @with_processes(["mapsd"]) @pytest.mark.tici def test_render_time_distribution(self): self._setup_test() diff --git a/selfdrive/test/setup_xvfb.sh b/selfdrive/test/setup_xvfb.sh new file mode 100755 index 0000000000..2c79adec4a --- /dev/null +++ b/selfdrive/test/setup_xvfb.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Sets up a virtual display for running map renderer and simulator without an X11 display + +DISP_ID=99 +export DISPLAY=:$DISP_ID + +sudo Xvfb $DISPLAY -screen 0 2160x1080x24 & + +# check for x11 socket for the specified display ID +while [ ! -S /tmp/.X11-unix/X$DISP_ID ] +do + echo "Waiting for Xvfb..." + sleep 1 +done \ No newline at end of file