diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 31af7b9f0d..09ff4975eb 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -11,6 +11,8 @@ concurrency: cancel-in-progress: true env: + PYTHONWARNINGS: error + BASE_IMAGE: openpilot-base CL_BASE_IMAGE: openpilot-base-cl DOCKER_REGISTRY: ghcr.io/commaai @@ -20,11 +22,11 @@ env: BUILD: | DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base . - RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v ~/scons_cache:/tmp/scons_cache -v ~/comma_download_cache:/tmp/comma_download_cache -v ~/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c + RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v ~/scons_cache:/tmp/scons_cache -v ~/comma_download_cache:/tmp/comma_download_cache -v ~/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c BUILD_CL: | DOCKER_BUILDKIT=1 docker build --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $CL_BASE_IMAGE:latest -f Dockerfile.openpilot_base_cl . - RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v ~/scons_cache:/tmp/scons_cache -v ~/comma_download_cache:/tmp/comma_download_cache -v ~/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c + RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v ~/scons_cache:/tmp/scons_cache -v ~/comma_download_cache:/tmp/comma_download_cache -v ~/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c UNIT_TEST: coverage run --append -m unittest discover @@ -66,7 +68,7 @@ jobs: cp pyproject.toml $STRIPPED_DIR cp poetry.lock $STRIPPED_DIR cd $STRIPPED_DIR - ${{ env.RUN }} "pre-commit run --all" + ${{ env.RUN }} "unset PYTHONWARNINGS && pre-commit run --all" build_all: name: build all @@ -142,6 +144,9 @@ jobs: - name: Install dependencies if: steps.dependency-cache.outputs.cache-hit != 'true' run: ./tools/mac_setup.sh + env: + # package install has DeprecationWarnings + PYTHONWARNINGS: default - name: Build openpilot run: | source tools/openpilot_env.sh @@ -214,7 +219,7 @@ jobs: run: eval "$BUILD" - name: pre-commit timeout-minutes: 4 - run: ${{ env.RUN }} "pre-commit run --all" + run: ${{ env.RUN }} "unset PYTHONWARNINGS && pre-commit run --all" valgrind: name: valgrind @@ -337,15 +342,18 @@ jobs: - name: Build openpilot run: | ${{ env.RUN }} "scons -j$(nproc)" + # PYTHONWARNINGS triggers a SyntaxError in onnxruntime - name: Run model replay with ONNX timeout-minutes: 2 run: | - ${{ env.RUN_CL }} "ONNXCPU=1 CI=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \ + ${{ env.RUN_CL }} "unset PYTHONWARNINGS && \ + ONNXCPU=1 CI=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \ coverage xml" - name: Run unit tests timeout-minutes: 4 run: | - ${{ env.RUN_CL }} "$UNIT_TEST selfdrive/modeld && \ + ${{ env.RUN_CL }} "unset PYTHONWARNINGS && \ + $UNIT_TEST selfdrive/modeld && \ coverage xml" - name: "Upload coverage to Codecov" uses: codecov/codecov-action@v3 diff --git a/Jenkinsfile b/Jenkinsfile index 6a834288c2..acf8a2e4cc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,6 +6,7 @@ ssh -tt -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' /usr/bin/bash < set -e export CI=1 +export PYTHONWARNINGS=error export LOGPRINT=debug export TEST_DIR=${env.TEST_DIR} export SOURCE_DIR=${env.SOURCE_DIR} @@ -65,6 +66,7 @@ pipeline { agent none environment { CI = "1" + PYTHONWARNINGS = "error" TEST_DIR = "/data/openpilot" SOURCE_DIR = "/data/openpilot_source/" AZURE_TOKEN = credentials('azure_token') diff --git a/tools/lib/framereader.py b/tools/lib/framereader.py index 50a1d3c245..b4845b9f0d 100644 --- a/tools/lib/framereader.py +++ b/tools/lib/framereader.py @@ -80,7 +80,7 @@ def vidindex(fn, typ): vidindex_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "vidindex") vidindex = os.path.join(vidindex_dir, "vidindex") - subprocess.check_call(["make"], cwd=vidindex_dir, stdout=open("/dev/null", "w")) + subprocess.check_call(["make"], cwd=vidindex_dir, stdout=subprocess.DEVNULL) with tempfile.NamedTemporaryFile() as prefix_f, \ tempfile.NamedTemporaryFile() as index_f: @@ -237,25 +237,23 @@ def decompress_video_data(rawdat, vid_fmt, w, h, pix_fmt): threads = os.getenv("FFMPEG_THREADS", "0") cuda = os.getenv("FFMPEG_CUDA", "0") == "1" - proc = subprocess.Popen( - ["ffmpeg", - "-threads", threads, - "-hwaccel", "none" if not cuda else "cuda", - "-c:v", "hevc", - "-vsync", "0", - "-f", vid_fmt, - "-flags2", "showall", - "-i", "pipe:0", - "-threads", threads, - "-f", "rawvideo", - "-pix_fmt", pix_fmt, - "pipe:1"], - stdin=tmpf, stdout=subprocess.PIPE, stderr=open("/dev/null")) - - # dat = proc.communicate()[0] - dat = proc.stdout.read() - if proc.wait() != 0: - raise DataUnreadableError("ffmpeg failed") + args = ["ffmpeg", + "-threads", threads, + "-hwaccel", "none" if not cuda else "cuda", + "-c:v", "hevc", + "-vsync", "0", + "-f", vid_fmt, + "-flags2", "showall", + "-i", "pipe:0", + "-threads", threads, + "-f", "rawvideo", + "-pix_fmt", pix_fmt, + "pipe:1"] + with subprocess.Popen(args, stdin=tmpf, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) as proc: + # dat = proc.communicate()[0] + dat = proc.stdout.read() + if proc.wait() != 0: + raise DataUnreadableError("ffmpeg failed") if pix_fmt == "rgb24": ret = np.frombuffer(dat, dtype=np.uint8).reshape(-1, h, w, 3)