diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index aba630f67c..6ab55f6add 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -27,7 +27,6 @@ env: RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 - XDIST: -n auto --dist=loadscope jobs: build_release: @@ -58,7 +57,7 @@ jobs: run: | cd $STRIPPED_DIR ${{ env.RUN }} "release/check-dirty.sh && \ - MAX_EXAMPLES=5 $PYTEST $XDIST selfdrive/car" + MAX_EXAMPLES=5 $PYTEST selfdrive/car" - name: pre-commit timeout-minutes: 3 run: | @@ -176,7 +175,7 @@ jobs: - name: Run unit tests timeout-minutes: 15 run: | - ${{ env.RUN }} "$PYTEST $XDIST --timeout 30 -o cpp_files=test_* -m 'not slow' && \ + ${{ env.RUN }} "$PYTEST --timeout 30 -m 'not slow' && \ ./selfdrive/ui/tests/create_test_translations.sh && \ QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \ ./selfdrive/ui/tests/test_translations.py && \ @@ -258,7 +257,7 @@ jobs: - name: Run regen timeout-minutes: 30 run: | - ${{ env.RUN_CL }} "ONNXCPU=1 $PYTEST $XDIST selfdrive/test/process_replay/test_regen.py && \ + ${{ env.RUN_CL }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \ chmod -R 777 /tmp/comma_download_cache" test_modeld: @@ -318,7 +317,7 @@ jobs: - name: Test car models timeout-minutes: 25 run: | - ${{ env.RUN }} "$PYTEST $XDIST selfdrive/car/tests/test_models.py && \ + ${{ env.RUN }} "$PYTEST selfdrive/car/tests/test_models.py && \ chmod -R 777 /tmp/comma_download_cache" env: NUM_JOBS: 5 diff --git a/Jenkinsfile b/Jenkinsfile index 6857ce36e1..1e1d485b10 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,6 +14,8 @@ export GIT_BRANCH=${env.GIT_BRANCH} export GIT_COMMIT=${env.GIT_COMMIT} export AZURE_TOKEN='${env.AZURE_TOKEN}' export MAPBOX_TOKEN='${env.MAPBOX_TOKEN}' +export PYTEST_ADDOPTS="-c selfdrive/test/pytest-tici.ini --rootdir ." + export GIT_SSH_COMMAND="ssh -i /data/gitkey" @@ -159,7 +161,7 @@ node { ["build openpilot", "cd selfdrive/manager && ./build.py"], ["check dirty", "release/check-dirty.sh"], ["onroad tests", "pytest selfdrive/test/test_onroad.py -s"], - ["time to onroad", "cd selfdrive/test/ && pytest test_time_to_onroad.py"], + ["time to onroad", "pytest selfdrive/test/test_time_to_onroad.py"], ]) }, 'HW + Unit Tests': { @@ -194,17 +196,17 @@ node { 'sensord': { deviceStage("LSM + MMC", "tici-lsmc", ["UNSAFE=1"], [ ["build", "cd selfdrive/manager && ./build.py"], - ["test sensord", "cd system/sensord/tests && pytest test_sensord.py"], + ["test sensord", "pytest system/sensord/tests/test_sensord.py"], ]) deviceStage("BMX + LSM", "tici-bmx-lsm", ["UNSAFE=1"], [ ["build", "cd selfdrive/manager && ./build.py"], - ["test sensord", "cd system/sensord/tests && pytest test_sensord.py"], + ["test sensord", "pytest system/sensord/tests/test_sensord.py"], ]) }, 'replay': { deviceStage("tici", "tici-replay", ["UNSAFE=1"], [ ["build", "cd selfdrive/manager && ./build.py"], - ["model replay", "cd selfdrive/test/process_replay && ./model_replay.py"], + ["model replay", "selfdrive/test/process_replay/model_replay.py"], ]) }, 'tizi': { @@ -231,7 +233,7 @@ node { sh label: "build", script: "selfdrive/manager/build.py" sh label: "test_models.py", script: "INTERNAL_SEG_CNT=250 INTERNAL_SEG_LIST=selfdrive/car/tests/test_models_segs.txt FILEREADER_CACHE=1 \ pytest -n42 --dist=loadscope selfdrive/car/tests/test_models.py" - sh label: "test_car_interfaces.py", script: "MAX_EXAMPLES=100 pytest -n42 selfdrive/car/tests/test_car_interfaces.py" + sh label: "test_car_interfaces.py", script: "MAX_EXAMPLES=100 pytest -n42 --dist=load selfdrive/car/tests/test_car_interfaces.py" } }, diff --git a/pyproject.toml b/pyproject.toml index 16dad2c643..4e494e65fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.pytest.ini_options] minversion = "6.0" -addopts = "--ignore=openpilot/ --ignore=cereal/ --ignore=opendbc/ --ignore=panda/ --ignore=rednose_repo/ --ignore=tinygrad_repo/ --ignore=laika_repo/ -Werror --strict-config --strict-markers --durations=10" -#cpp_files = "test_*" # uncomment when agnos has pytest-cpp and remove from CI +addopts = "--ignore=openpilot/ --ignore=cereal/ --ignore=opendbc/ --ignore=panda/ --ignore=rednose_repo/ --ignore=tinygrad_repo/ --ignore=laika_repo/ -Werror --strict-config --strict-markers --durations=10 -n auto --dist=loadscope" +cpp_files = "test_*" python_files = "test_*.py" #timeout = "30" # you get this long by default markers = [ diff --git a/release/files_common b/release/files_common index 5a076f2b1a..b2623e0113 100644 --- a/release/files_common +++ b/release/files_common @@ -293,6 +293,7 @@ selfdrive/test/helpers.py selfdrive/test/setup_device_ci.sh selfdrive/test/test_onroad.py selfdrive/test/test_time_to_onroad.py +selfdrive/test/pytest-tici.ini selfdrive/ui/.gitignore selfdrive/ui/SConscript diff --git a/selfdrive/test/pytest-tici.ini b/selfdrive/test/pytest-tici.ini new file mode 100644 index 0000000000..a553018309 --- /dev/null +++ b/selfdrive/test/pytest-tici.ini @@ -0,0 +1,5 @@ +[pytest] +addopts = -Werror --strict-config --strict-markers +markers = + slow: tests that take awhile to run and can be skipped with -m 'not slow' + tici: tests that are only meant to run on the C3/C3X diff --git a/system/sensord/tests/test_sensord.py b/system/sensord/tests/test_sensord.py index 0905248071..3075c8a343 100755 --- a/system/sensord/tests/test_sensord.py +++ b/system/sensord/tests/test_sensord.py @@ -106,7 +106,7 @@ class TestSensord(unittest.TestCase): os.environ["LSM_SELF_TEST"] = "1" # read initial sensor values every test case can use - os.system("pkill -f ./sensord") + os.system("pkill -f \\\\./sensord") try: managed_processes["sensord"].start() cls.sample_secs = int(os.getenv("SAMPLE_SECS", "10"))