diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 8c46e15c2f..15b358eb10 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -61,13 +61,13 @@ jobs: cd $STRIPPED_DIR ${{ env.RUN }} "release/check-dirty.sh && \ MAX_EXAMPLES=5 $PYTEST -m 'not slow' selfdrive/car" - - name: static analysis + - name: Static analysis timeout-minutes: 1 run: | cd $GITHUB_WORKSPACE cp pyproject.toml $STRIPPED_DIR cd $STRIPPED_DIR - ${{ env.RUN }} "scripts/lint.sh" + ${{ env.RUN }} "scripts/lint.sh --skip check_added_large_files" build: strategy: @@ -140,16 +140,15 @@ jobs: runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }} + env: + PYTHONWARNINGS: default steps: - uses: actions/checkout@v4 - with: - submodules: true - - uses: ./.github/workflows/setup-with-retry - - name: Build openpilot - run: ${{ env.RUN }} "scons -j$(nproc)" - - name: static analysis + - name: Setup + run: tools/op.sh setup + - name: Static analysis timeout-minutes: 1 - run: ${{ env.RUN }} "scripts/lint.sh" + run: tools/op.sh lint unit_tests: name: unit tests diff --git a/pyproject.toml b/pyproject.toml index b311a854dc..a0131773e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,6 +88,7 @@ testing = [ "pytest-repeat", "ruff", "codespell", + "pre-commit-hooks", ] dev = [ @@ -169,7 +170,7 @@ quiet-level = 3 # if you've got a short variable name that's getting flagged, add it here ignore-words-list = "bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,preints,whit,indexIn,ws,uint,grey,deque,stdio,amin,BA,LITE,atEnd,UIs,errorString,arange,FocusIn,od,tim,relA,hist,copyable,jupyter,thead" builtin = "clear,rare,informal,code,names,en-GB_to_en-US" -skip = "./third_party/*, ./tinygrad/*, ./tinygrad_repo/*, ./msgq/*, ./panda/*, ./opendbc/*, ./opendbc_repo/*, ./rednose/*, ./rednose_repo/*, ./teleoprtc/*, ./teleoprtc_repo/*, ./selfdrive/ui/translations/*.ts, uv.lock, *.onnx, ./cereal/gen/*, */c_generated_code/*" +skip = "./third_party/*, ./tinygrad/*, ./tinygrad_repo/*, ./msgq/*, ./panda/*, ./opendbc/*, ./opendbc_repo/*, ./rednose/*, ./rednose_repo/*, ./teleoprtc/*, ./teleoprtc_repo/*, *.ts, uv.lock, *.onnx, ./cereal/gen/*, */c_generated_code/*" [tool.mypy] python_version = "3.11" @@ -177,9 +178,11 @@ plugins = [ "numpy.typing.mypy_plugin", ] exclude = [ - "body/", "cereal/", + "msgq/", + "msgq_repo/", "opendbc/", + "opendbc_repo/", "panda/", "rednose/", "rednose_repo/", @@ -204,6 +207,10 @@ warn_return_any=true # allow implicit optionals for default args implicit_optional = true +local_partial_types=true +explicit_package_bases=true +disable_error_code = "annotation-unchecked" + # https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml [tool.ruff] indent-width = 2 diff --git a/scripts/lint.sh b/scripts/lint.sh index 3da79f62f4..38dabfbf31 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -1,11 +1,99 @@ #!/bin/bash set -e +RED='\033[0;31m' +GREEN='\033[0;32m' +UNDERLINE='\033[4m' +BOLD='\033[1m' +NC='\033[0m' + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" cd $DIR/../ -# TODO: bring back rest of pre-commit checks: -# https://github.com/commaai/openpilot/blob/4b11c9e914707df9def598616995be2a5d355a6a/.pre-commit-config.yaml#L2 +FAILED=0 + +IGNORED_FILES="uv\.lock|docs\/CARS.md" +IGNORED_DIRS="^third_party.*|^msgq.*|^msgq_repo.*|^opendbc.*|^opendbc_repo.*|^cereal.*|^panda.*|^rednose.*|^rednose_repo.*|^tinygrad.*|^tinygrad_repo.*|^teleoprtc.*|^teleoprtc_repo.*" + +function run() { + shopt -s extglob + case $1 in + $SKIP ) return 0 ;; + esac + + echo -en "$1" + + for ((i=0; i<$((50 - ${#1})); i++)); do + echo -n "." + done + + shift 1; + CMD="$@" + + set +e + log="$((eval "$CMD" ) 2>&1)" + + if [[ $? -eq 0 ]]; then + echo -e "[${GREEN}✔${NC}]" + else + echo -e "[${RED}✗${NC}]" + echo "$log" + FAILED=1 + fi + set -e +} + +function run_tests() { + ALL_FILES=$1 + PYTHON_FILES=$2 + + run "ruff" ruff check $PYTHON_FILES --quiet + run "lint-imports" lint-imports + run "check_added_large_files" python3 -m pre_commit_hooks.check_added_large_files --enforce-all $ALL_FILES --maxkb=120 + run "check_shebang_scripts_are_executable" python3 -m pre_commit_hooks.check_shebang_scripts_are_executable $ALL_FILES + + if [[ -z "$FAST" ]]; then + run "mypy" mypy $PYTHON_FILES + run "codespell" codespell $ALL_FILES + fi + + return $FAILED +} + +function help() { + echo "A fast linter" + echo "" + echo -e "${BOLD}${UNDERLINE}Usage:${NC} op lint [OPTIONS]" + echo "" + echo -e "${BOLD}${UNDERLINE}Options:${NC}" + echo -e " ${BOLD}-f, --fast${NC}" + echo " Skip slow tests" + echo -e " ${BOLD}-s, --skip${NC}" + echo " Specify a test to skip" + echo "" + echo -e "${BOLD}${UNDERLINE}Examples:${NC}" + echo " op lint --skip mypy --skip ruff" + echo " Skip the ruff and mypy check" +} + +while true; do + case $1 in + -f | --fast ) shift 1; FAST="1" ;; + -s | --skip ) shift 1; SKIP+="$1|"; shift 1 ;; + -h | --help | -help | --h ) help; exit 0 ;; + * ) break ;; + esac +done + +SKIP="@($(echo $SKIP | sed 's/|$//'))" + +GIT_FILES="$(git ls-files | sed -E "s/$IGNORED_FILES|$IGNORED_DIRS//g")" +ALL_FILES="" +for f in $GIT_FILES; do + if [[ -f $f ]]; then + ALL_FILES+="$f"$'\n' + fi +done +PYTHON_FILES=$(echo "$ALL_FILES" | grep --color=never '.py$' || true) -ruff check . -lint-imports +run_tests "$ALL_FILES" "$PYTHON_FILES" diff --git a/uv.lock b/uv.lock index bca0895ebf..b5a734f300 100644 --- a/uv.lock +++ b/uv.lock @@ -1454,6 +1454,7 @@ testing = [ { name = "hypothesis", marker = "python_version == '3.11' or python_version >= '3.12' or (python_version < '3.12' and (python_version < '3.11' or python_version > '3.11'))" }, { name = "import-linter", marker = "python_version == '3.11' or python_version >= '3.12' or (python_version < '3.12' and (python_version < '3.11' or python_version > '3.11'))" }, { name = "mypy", marker = "python_version == '3.11' or python_version >= '3.12' or (python_version < '3.12' and (python_version < '3.11' or python_version > '3.11'))" }, + { name = "pre-commit-hooks", marker = "python_version == '3.11' or python_version >= '3.12' or (python_version < '3.12' and (python_version < '3.11' or python_version > '3.11'))" }, { name = "pytest", marker = "python_version == '3.11' or python_version >= '3.12' or (python_version < '3.12' and (python_version < '3.11' or python_version > '3.11'))" }, { name = "pytest-asyncio", marker = "python_version == '3.11' or python_version >= '3.12' or (python_version < '3.12' and (python_version < '3.11' or python_version > '3.11'))" }, { name = "pytest-cov", marker = "python_version == '3.11' or python_version >= '3.12' or (python_version < '3.12' and (python_version < '3.11' or python_version > '3.11'))" }, @@ -1637,6 +1638,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9b/fb/a70a4214956182e0d7a9099ab17d50bfcba1056188e9b14f35b9e2b62a0d/portalocker-2.10.1-py3-none-any.whl", hash = "sha256:53a5984ebc86a025552264b459b46a2086e269b21823cb572f8f28ee759e45bf", size = 18423 }, ] +[[distribution]] +name = "pre-commit-hooks" +version = "4.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ruamel-yaml", marker = "python_version == '3.11' or python_version >= '3.12' or (python_version < '3.12' and (python_version < '3.11' or python_version > '3.11'))" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/0d/b4e97cd99b26c0cd8265c9f19ee7e55248142f0c9955e4d119de96fa4a13/pre_commit_hooks-4.6.0.tar.gz", hash = "sha256:eb1f43ee67869cd41b4c59017fad4a0f9d4d61201d163f2135535aaf65035a2b", size = 29579 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/d8/d09b270248bc2441f966dd094a3de8ae813112d2687036f621202bdef80b/pre_commit_hooks-4.6.0-py2.py3-none-any.whl", hash = "sha256:a69199e6a2d45ec59c1020a81ca1549abddc2afb798276d9a0d951752d6abbfe", size = 41193 }, +] + [[distribution]] name = "progressbar" version = "2.5" @@ -4882,6 +4895,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/28/92423fe9673b738c180fb5b6b8ea4203fe4b02c1d20b06b7fae79d11cc24/rerun_sdk-0.17.0-cp38-abi3-win_amd64.whl", hash = "sha256:34e5595a326cbdddfebdf00b08e877358c564fce74cc8c6d617fc89ef3a6aa70", size = 29490986 }, ] +[[distribution]] +name = "ruamel-yaml" +version = "0.18.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ruamel-yaml-clib", marker = "python_version < '3.13' and platform_python_implementation == 'CPython'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/29/81/4dfc17eb6ebb1aac314a3eb863c1325b907863a1b8b1382cdffcb6ac0ed9/ruamel.yaml-0.18.6.tar.gz", hash = "sha256:8b27e6a217e786c6fbe5634d8f3f11bc63e0f80f6a5890f28863d9c45aac311b", size = 143362 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/67/8ece580cc363331d9a53055130f86b096bf16e38156e33b1d3014fffda6b/ruamel.yaml-0.18.6-py3-none-any.whl", hash = "sha256:57b53ba33def16c4f3d807c0ccbc00f8a6081827e81ba2491691b76882d0c636", size = 117761 }, +] + +[[distribution]] +name = "ruamel-yaml-clib" +version = "0.2.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/ab/bab9eb1566cd16f060b54055dd39cf6a34bfa0240c53a7218c43e974295b/ruamel.yaml.clib-0.2.8.tar.gz", hash = "sha256:beb2e0404003de9a4cab9753a8805a8fe9320ee6673136ed7f04255fe60bb512", size = 213824 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/15/971b385c098e8d0d170893f5ba558452bb7b776a0c90658b8f4dd0e3382b/ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:bef08cd86169d9eafb3ccb0a39edb11d8e25f3dae2b28f5c52fd997521133069", size = 148870 }, + { url = "https://files.pythonhosted.org/packages/01/b0/4ddef56e9f703d7909febc3a421d709a3482cda25826816ec595b73e3847/ruamel.yaml.clib-0.2.8-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:b16420e621d26fdfa949a8b4b47ade8810c56002f5389970db4ddda51dbff248", size = 134475 }, + { url = "https://files.pythonhosted.org/packages/a4/f7/22d6b620ed895a05d40802d8281eff924dc6190f682d933d4efff60db3b5/ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:25c515e350e5b739842fc3228d662413ef28f295791af5e5110b543cf0b57d9b", size = 544020 }, + { url = "https://files.pythonhosted.org/packages/7c/e4/0d19d65e340f93df1c47f323d95fa4b256bb28320290f5fddef90837853a/ruamel.yaml.clib-0.2.8-cp311-cp311-manylinux_2_24_aarch64.whl", hash = "sha256:1707814f0d9791df063f8c19bb51b0d1278b8e9a2353abbb676c2f685dee6afe", size = 642643 }, + { url = "https://files.pythonhosted.org/packages/c9/ff/f781eb5e2ae011e586d5426e2086a011cf1e0f59704a6cad1387975c5a62/ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:46d378daaac94f454b3a0e3d8d78cafd78a026b1d71443f4966c696b48a6d899", size = 695832 }, + { url = "https://files.pythonhosted.org/packages/e3/41/f62e67ac651358b8f0d60cfb12ab2daf99b1b69eeaa188d0cec809d943a6/ruamel.yaml.clib-0.2.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:09b055c05697b38ecacb7ac50bdab2240bfca1a0c4872b0fd309bb07dc9aa3a9", size = 730923 }, + { url = "https://files.pythonhosted.org/packages/9f/f0/19ab8acbf983cd1b37f47d27ceb8b10a738d60d36316a54bad57e0d73fbb/ruamel.yaml.clib-0.2.8-cp311-cp311-win32.whl", hash = "sha256:53a300ed9cea38cf5a2a9b069058137c2ca1ce658a874b79baceb8f892f915a7", size = 99999 }, + { url = "https://files.pythonhosted.org/packages/ec/54/d8a795997921d87224c65d44499ca595a833093fb215b133f920c1062956/ruamel.yaml.clib-0.2.8-cp311-cp311-win_amd64.whl", hash = "sha256:c2a72e9109ea74e511e29032f3b670835f8a59bbdc9ce692c5b4ed91ccf1eedb", size = 118008 }, + { url = "https://files.pythonhosted.org/packages/7a/a2/eb5e9d088cb9d15c24d956944c09dca0a89108ad6e2e913c099ef36e3f0d/ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:ebc06178e8821efc9692ea7544aa5644217358490145629914d8020042c24aa1", size = 144636 }, + { url = "https://files.pythonhosted.org/packages/66/98/8de4f22bbfd9135deb3422e96d450c4bc0a57d38c25976119307d2efe0aa/ruamel.yaml.clib-0.2.8-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:edaef1c1200c4b4cb914583150dcaa3bc30e592e907c01117c08b13a07255ec2", size = 135684 }, + { url = "https://files.pythonhosted.org/packages/30/d3/5fe978cd01a61c12efd24d65fa68c6f28f28c8073a06cf11db3a854390ca/ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d176b57452ab5b7028ac47e7b3cf644bcfdc8cacfecf7e71759f7f51a59e5c92", size = 734571 }, + { url = "https://files.pythonhosted.org/packages/55/b3/e2531a050758b717c969cbf76c103b75d8a01e11af931b94ba656117fbe9/ruamel.yaml.clib-0.2.8-cp312-cp312-manylinux_2_24_aarch64.whl", hash = "sha256:1dc67314e7e1086c9fdf2680b7b6c2be1c0d8e3a8279f2e993ca2a7545fecf62", size = 643946 }, + { url = "https://files.pythonhosted.org/packages/0d/aa/06db7ca0995b513538402e11280282c615b5ae5f09eb820460d35fb69715/ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3213ece08ea033eb159ac52ae052a4899b56ecc124bb80020d9bbceeb50258e9", size = 692169 }, + { url = "https://files.pythonhosted.org/packages/27/38/4cf4d482b84ecdf51efae6635cc5483a83cf5ca9d9c13e205a750e251696/ruamel.yaml.clib-0.2.8-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aab7fd643f71d7946f2ee58cc88c9b7bfc97debd71dcc93e03e2d174628e7e2d", size = 740325 }, + { url = "https://files.pythonhosted.org/packages/6f/67/c62c6eea53a4feb042727a3d6c18f50dc99683c2b199c06bd2a9e3db8e22/ruamel.yaml.clib-0.2.8-cp312-cp312-win32.whl", hash = "sha256:5c365d91c88390c8d0a8545df0b5857172824b1c604e867161e6b3d59a827eaa", size = 98639 }, + { url = "https://files.pythonhosted.org/packages/10/d2/52a3d810d0b5b3720725c0504a27b3fced7b6f310fe928f7019d79387bc1/ruamel.yaml.clib-0.2.8-cp312-cp312-win_amd64.whl", hash = "sha256:1758ce7d8e1a29d23de54a16ae867abd370f01b5a69e1a3ba75223eaa3ca1a1b", size = 115305 }, +] + [[distribution]] name = "rubicon-objc" version = "0.4.9"