diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 4f7e57b5ff..e73e0df2a8 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -53,7 +53,7 @@ jobs: run: | cd $STRIPPED_DIR ${{ env.RUN }} "release/check-dirty.sh && \ - MAX_EXAMPLES=5 $PYTEST selfdrive/car" + MAX_EXAMPLES=5 $PYTEST -m 'not slow' selfdrive/car" - name: pre-commit timeout-minutes: 3 run: | @@ -62,7 +62,7 @@ jobs: cp pyproject.toml $STRIPPED_DIR cp poetry.lock $STRIPPED_DIR cd $STRIPPED_DIR - ${{ env.RUN }} "unset PYTHONWARNINGS && SKIP=check-added-large-files pre-commit run --all && chmod -R 777 /tmp/pre-commit" + ${{ env.RUN }} "unset PYTHONWARNINGS && SKIP=check-added-large-files,check-hooks-apply,check-useless-excludes pre-commit run --all && chmod -R 777 /tmp/pre-commit" build: strategy: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 190425285d..28f8acd6e2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,7 +47,7 @@ repos: args: - --local-partial-types - --explicit-package-bases - exclude: '^(third_party/)|(cereal/)|(opendbc/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)' + exclude: '^(third_party/)|(body/)|(cereal/)|(opendbc/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)' - repo: local hooks: - id: cppcheck diff --git a/release/build_devel.sh b/release/build_devel.sh index 8b6816e423..18d99bb1cf 100755 --- a/release/build_devel.sh +++ b/release/build_devel.sh @@ -1,5 +1,4 @@ #!/usr/bin/bash - set -ex DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" @@ -44,11 +43,12 @@ git clean -xdff # do the files copy echo "[-] copying files T=$SECONDS" cd $SOURCE_DIR -cp -pR --parents $(cat release/files_*) $TARGET_DIR/ +cp -pR --parents $(./release/release_files.py) $TARGET_DIR/ # in the directory cd $TARGET_DIR rm -f panda/board/obj/panda.bin.signed +git submodule status # include source commit hash and build date in commit GIT_HASH=$(git --git-dir=$SOURCE_DIR/.git rev-parse HEAD) diff --git a/release/build_release.sh b/release/build_release.sh index b381ff7325..c2745ea287 100755 --- a/release/build_release.sh +++ b/release/build_release.sh @@ -9,13 +9,6 @@ cd $DIR BUILD_DIR=/data/openpilot SOURCE_DIR="$(git rev-parse --show-toplevel)" -if [ -f /TICI ]; then - FILES_SRC="release/files_tici" -else - echo "no release files set" - exit 1 -fi - if [ -z "$RELEASE_BRANCH" ]; then echo "RELEASE_BRANCH is not set" exit 1 @@ -36,8 +29,7 @@ git checkout --orphan $RELEASE_BRANCH # do the files copy echo "[-] copying files T=$SECONDS" cd $SOURCE_DIR -cp -pR --parents $(cat release/files_common) $BUILD_DIR/ -cp -pR --parents $(cat $FILES_SRC) $BUILD_DIR/ +cp -pR --parents $(./release/release_files.py) $TARGET_DIR/ # in the directory cd $BUILD_DIR @@ -77,6 +69,10 @@ find . -name '__pycache__' -delete rm -rf .sconsign.dblite Jenkinsfile release/ rm selfdrive/modeld/models/supercombo.onnx +find third_party/ -name '*x86*' -exec rm -r {} + +find third_party/ -name '*Darwin*' -exec rm -r {} + + + # Restore third_party git checkout third_party/ diff --git a/release/files_common b/release/files_common deleted file mode 100644 index cc797fb20b..0000000000 --- a/release/files_common +++ /dev/null @@ -1,560 +0,0 @@ -.gitignore -LICENSE -launch_env.sh -launch_chffrplus.sh -launch_openpilot.sh - -Jenkinsfile -SConstruct -pyproject.toml - -README.md -RELEASES.md -docs/CARS.md -docs/CONTRIBUTING.md -docs/INTEGRATION.md -docs/LIMITATIONS.md -site_scons/site_tools/cython.py - -openpilot/__init__.py -openpilot/** - -common/.gitignore -common/__init__.py -common/*.py -common/*.pyx -common/mock/* - -common/transformations/__init__.py -common/transformations/camera.py -common/transformations/model.py - -common/transformations/SConscript -common/transformations/coordinates.py -common/transformations/coordinates.cc -common/transformations/coordinates.hpp -common/transformations/orientation.py -common/transformations/orientation.cc -common/transformations/orientation.hpp -common/transformations/transformations.pxd -common/transformations/transformations.pyx - -common/api/__init__.py - -release/* - -tools/__init__.py -tools/lib/* -tools/bodyteleop/.gitignore -tools/bodyteleop/web.py -tools/bodyteleop/static/* -tools/joystick/* -tools/replay/*.cc -tools/replay/*.h - -selfdrive/__init__.py - -system/logmessaged.py -system/micd.py -system/version.py - -selfdrive/SConscript - -system/athena/__init__.py -system/athena/athenad.py -system/athena/manage_athenad.py -system/athena/registration.py - -selfdrive/boardd/.gitignore -selfdrive/boardd/SConscript -selfdrive/boardd/__init__.py -selfdrive/boardd/boardd.cc -selfdrive/boardd/boardd.h -selfdrive/boardd/main.cc -selfdrive/boardd/boardd.py -selfdrive/boardd/boardd_api_impl.pyx -selfdrive/boardd/can_list_to_can_capnp.cc -selfdrive/boardd/panda.cc -selfdrive/boardd/panda.h -selfdrive/boardd/spi.cc -selfdrive/boardd/panda_comms.h -selfdrive/boardd/panda_comms.cc -selfdrive/boardd/pandad.py -selfdrive/boardd/tests/test_boardd_loopback.py - -selfdrive/car/__init__.py -selfdrive/car/card.py -selfdrive/car/docs_definitions.py -selfdrive/car/car_helpers.py -selfdrive/car/fingerprints.py -selfdrive/car/interfaces.py -selfdrive/car/values.py -selfdrive/car/vin.py -selfdrive/car/disable_ecu.py -selfdrive/car/fw_versions.py -selfdrive/car/fw_query_definitions.py -selfdrive/car/ecu_addrs.py -selfdrive/car/isotp_parallel_query.py -selfdrive/car/tests/__init__.py -selfdrive/car/tests/test_car_interfaces.py -selfdrive/car/torque_data/* - -selfdrive/car/body/*.py -selfdrive/car/chrysler/*.py -selfdrive/car/ford/*.py -selfdrive/car/gm/*.py -selfdrive/car/honda/*.py -selfdrive/car/hyundai/*.py -selfdrive/car/mazda/*.py -selfdrive/car/mock/*.py -selfdrive/car/nissan/*.py -selfdrive/car/subaru/*.py -selfdrive/car/tesla/*.py -selfdrive/car/toyota/*.py -selfdrive/car/volkswagen/*.py - -selfdrive/debug/can_printer.py -selfdrive/debug/check_freq.py -selfdrive/debug/dump.py -selfdrive/debug/filter_log_message.py -selfdrive/debug/format_fingerprints.py -selfdrive/debug/get_fingerprint.py -selfdrive/debug/uiview.py - -selfdrive/debug/hyundai_enable_radar_points.py -selfdrive/debug/vw_mqb_config.py - -common/SConscript -common/version.h - -common/*.h -common/*.cc - -selfdrive/controls/__init__.py -selfdrive/controls/controlsd.py -selfdrive/controls/plannerd.py -selfdrive/controls/radard.py -selfdrive/controls/lib/__init__.py -selfdrive/controls/lib/alertmanager.py -selfdrive/controls/lib/alerts_offroad.json -selfdrive/controls/lib/desire_helper.py -selfdrive/controls/lib/drive_helpers.py -selfdrive/controls/lib/events.py -selfdrive/controls/lib/latcontrol_angle.py -selfdrive/controls/lib/latcontrol_torque.py -selfdrive/controls/lib/latcontrol_pid.py -selfdrive/controls/lib/latcontrol.py -selfdrive/controls/lib/longcontrol.py -selfdrive/controls/lib/longitudinal_planner.py -selfdrive/controls/lib/pid.py -selfdrive/controls/lib/vehicle_model.py - -selfdrive/controls/lib/lateral_mpc_lib/.gitignore -selfdrive/controls/lib/longitudinal_mpc_lib/.gitignore -selfdrive/controls/lib/lateral_mpc_lib/* -selfdrive/controls/lib/longitudinal_mpc_lib/* - -system/__init__.py -system/*.py - -system/sentry.py -system/tombstoned.py -system/statsd.py - -system/hardware/__init__.py -system/hardware/base.h -system/hardware/base.py -system/hardware/hw.h -system/hardware/hw.py -system/hardware/tici/__init__.py -system/hardware/tici/hardware.h -system/hardware/tici/hardware.py -system/hardware/tici/pins.py -system/hardware/tici/agnos.py -system/hardware/tici/agnos.json -system/hardware/tici/amplifier.py -system/hardware/tici/updater -system/hardware/tici/iwlist.py -system/hardware/tici/esim.nmconnection -system/hardware/pc/__init__.py -system/hardware/pc/hardware.h -system/hardware/pc/hardware.py - -system/ubloxd/.gitignore -system/ubloxd/SConscript -system/ubloxd/pigeond.py -system/ubloxd/generated/* -system/ubloxd/*.h -system/ubloxd/*.cc - -system/updated/** - -selfdrive/locationd/__init__.py -selfdrive/locationd/SConscript -selfdrive/locationd/.gitignore -selfdrive/locationd/locationd.h -selfdrive/locationd/locationd.cc -selfdrive/locationd/paramsd.py -selfdrive/locationd/models/__init__.py -selfdrive/locationd/models/.gitignore -selfdrive/locationd/models/car_kf.py -selfdrive/locationd/models/live_kf.py -selfdrive/locationd/models/live_kf.h -selfdrive/locationd/models/live_kf.cc -selfdrive/locationd/models/constants.py - -selfdrive/locationd/torqued.py -selfdrive/locationd/calibrationd.py -selfdrive/locationd/helpers.py - -system/logcatd/.gitignore -system/logcatd/SConscript -system/logcatd/logcatd_systemd.cc - -system/proclogd/SConscript -system/proclogd/main.cc -system/proclogd/proclog.cc -system/proclogd/proclog.h - -system/loggerd/.gitignore -system/loggerd/SConscript -system/loggerd/encoder/encoder.cc -system/loggerd/encoder/encoder.h -system/loggerd/encoder/v4l_encoder.cc -system/loggerd/encoder/v4l_encoder.h -system/loggerd/video_writer.cc -system/loggerd/video_writer.h -system/loggerd/logger.cc -system/loggerd/logger.h -system/loggerd/loggerd.cc -system/loggerd/loggerd.h -system/loggerd/encoderd.cc -system/loggerd/bootlog.cc -system/loggerd/encoder/ffmpeg_encoder.cc -system/loggerd/encoder/ffmpeg_encoder.h - -system/loggerd/__init__.py -system/loggerd/config.py -system/loggerd/uploader.py -system/loggerd/deleter.py -system/loggerd/xattr_cache.py - -system/sensord/.gitignore -system/sensord/SConscript -system/sensord/sensors_qcom2.cc -system/sensord/sensors/*.cc -system/sensord/sensors/*.h - -system/webrtc/__init__.py -system/webrtc/webrtcd.py -system/webrtc/schema.py -system/webrtc/device/audio.py -system/webrtc/device/video.py - -system/thermald/thermald.py -system/thermald/power_monitoring.py -system/thermald/fan_controller.py - -selfdrive/test/__init__.py -selfdrive/test/fuzzy_generation.py -selfdrive/test/helpers.py -selfdrive/test/setup_device_ci.sh -selfdrive/test/test_onroad.py -selfdrive/test/test_time_to_onroad.py - -selfdrive/ui/.gitignore -selfdrive/ui/SConscript -selfdrive/ui/*.cc -selfdrive/ui/*.h -selfdrive/ui/text -selfdrive/ui/spinner -selfdrive/ui/soundd.py -selfdrive/ui/translations/*.ts -selfdrive/ui/translations/languages.json -selfdrive/ui/update_translations.py -selfdrive/ui/tests/test_translations.py - -selfdrive/ui/qt/*.cc -selfdrive/ui/qt/*.h -selfdrive/ui/qt/network/*.cc -selfdrive/ui/qt/network/*.h -selfdrive/ui/qt/offroad/*.cc -selfdrive/ui/qt/offroad/*.h -selfdrive/ui/qt/offroad/*.qml -selfdrive/ui/qt/onroad/*.cc -selfdrive/ui/qt/onroad/*.h -selfdrive/ui/qt/widgets/*.cc -selfdrive/ui/qt/widgets/*.h -selfdrive/ui/qt/maps/*.cc -selfdrive/ui/qt/maps/*.h -selfdrive/ui/qt/setup/*.cc -selfdrive/ui/qt/setup/*.h - -selfdrive/ui/installer/*.cc -selfdrive/ui/installer/*.h -selfdrive/ui/installer/*.cc - -system/camerad/SConscript -system/camerad/main.cc - -system/camerad/snapshot/* -system/camerad/cameras/camera_common.h -system/camerad/cameras/camera_common.cc -system/camerad/sensors/*.h -system/camerad/sensors/*.cc - -system/manager/__init__.py -system/manager/** - -selfdrive/modeld/.gitignore -selfdrive/modeld/__init__.py -selfdrive/modeld/SConscript -selfdrive/modeld/modeld.py -selfdrive/modeld/parse_model_outputs.py -selfdrive/modeld/fill_model_msg.py -selfdrive/modeld/get_model_metadata.py -selfdrive/modeld/dmonitoringmodeld.py -selfdrive/modeld/constants.py -selfdrive/modeld/modeld - -selfdrive/modeld/models/__init__.py -selfdrive/modeld/models/*.pxd -selfdrive/modeld/models/*.pyx - -selfdrive/modeld/models/commonmodel.cc -selfdrive/modeld/models/commonmodel.h - -selfdrive/modeld/models/supercombo.onnx - -selfdrive/modeld/models/dmonitoring_model_q.dlc - -selfdrive/modeld/transforms/loadyuv.cc -selfdrive/modeld/transforms/loadyuv.h -selfdrive/modeld/transforms/loadyuv.cl -selfdrive/modeld/transforms/transform.cc -selfdrive/modeld/transforms/transform.h -selfdrive/modeld/transforms/transform.cl - -selfdrive/modeld/thneed/*.py -selfdrive/modeld/thneed/thneed.h -selfdrive/modeld/thneed/thneed_common.cc -selfdrive/modeld/thneed/thneed_qcom2.cc -selfdrive/modeld/thneed/serialize.cc - -selfdrive/modeld/runners/__init__.py -selfdrive/modeld/runners/*.pxd -selfdrive/modeld/runners/*.pyx -selfdrive/modeld/runners/*.cc -selfdrive/modeld/runners/*.h -selfdrive/modeld/runners/*.py - -selfdrive/monitoring/dmonitoringd.py -selfdrive/monitoring/helpers.py - -selfdrive/navd/.gitignore -selfdrive/navd/__init__.py -selfdrive/navd/** - -selfdrive/assets/.gitignore -selfdrive/assets/assets.qrc -selfdrive/assets/*.png -selfdrive/assets/*.svg -selfdrive/assets/body/* -selfdrive/assets/fonts/*.ttf -selfdrive/assets/icons/* -selfdrive/assets/images/* -selfdrive/assets/offroad/* -selfdrive/assets/sounds/* -selfdrive/assets/training/* -selfdrive/assets/navigation/* - -third_party/.gitignore -third_party/SConscript - -third_party/linux/** -third_party/opencl/** - -third_party/json11/json11.cpp -third_party/json11/json11.hpp - -third_party/qrcode/*.cc -third_party/qrcode/*.hpp - -third_party/kaitai/*.h -third_party/kaitai/*.cpp - -third_party/libyuv/include/** - -third_party/snpe/include/** -third_party/snpe/dsp** - -third_party/acados/.gitignore -third_party/acados/include/** -third_party/acados/acados_template/** - -third_party/bootstrap/** -third_party/qt5/larch64/bin/** -third_party/maplibre-native-qt/** - -scripts/update_now.sh -scripts/stop_updater.sh - -teleoprtc/** - -rednose_repo/site_scons/site_tools/rednose_filter.py -rednose/.gitignore -rednose/** - -body/.gitignore -body/board/SConscript -body/board/*.h -body/board/*.c -body/board/*.s -body/board/*.ld -body/board/inc/** -body/board/obj/ -body/board/bldc/** -body/board/drivers/** -body/certs/** -body/crypto/** - -cereal/.gitignore -cereal/__init__.py -cereal/car.capnp -cereal/custom.capnp -cereal/legacy.capnp -cereal/log.capnp -cereal/services.py -cereal/SConscript -cereal/include/** -cereal/logger/logger.h -cereal/messaging/.gitignore -cereal/messaging/__init__.py -cereal/messaging/bridge.cc -cereal/messaging/event.cc -cereal/messaging/event.h -cereal/messaging/impl_fake.cc -cereal/messaging/impl_fake.h -cereal/messaging/impl_msgq.cc -cereal/messaging/impl_msgq.h -cereal/messaging/impl_zmq.cc -cereal/messaging/impl_zmq.h -cereal/messaging/messaging.cc -cereal/messaging/messaging.h -cereal/messaging/messaging.pxd -cereal/messaging/messaging_pyx.pyx -cereal/messaging/msgq.cc -cereal/messaging/msgq.h -cereal/messaging/socketmaster.cc -cereal/visionipc/.gitignore -cereal/visionipc/__init__.py -cereal/visionipc/*.cc -cereal/visionipc/*.h -cereal/visionipc/*.pyx -cereal/visionipc/*.pxd - -panda/.gitignore -panda/__init__.py -panda/SConscript -panda/board/** -panda/certs/** -panda/crypto/** -panda/examples/query_fw_versions.py -panda/python/** - -opendbc/.gitignore -opendbc/__init__.py -opendbc/can/__init__.py -opendbc/can/SConscript -opendbc/can/can_define.py -opendbc/can/common.cc -opendbc/can/common.h -opendbc/can/common.pxd -opendbc/can/common_dbc.h -opendbc/can/dbc.cc -opendbc/can/packer.cc -opendbc/can/packer.py -opendbc/can/packer_pyx.pyx -opendbc/can/parser.cc -opendbc/can/parser.py -opendbc/can/parser_pyx.pyx - -opendbc/comma_body.dbc - -opendbc/chrysler_ram_hd_generated.dbc -opendbc/chrysler_ram_dt_generated.dbc -opendbc/chrysler_pacifica_2017_hybrid_generated.dbc -opendbc/chrysler_pacifica_2017_hybrid_private_fusion.dbc - -opendbc/gm_global_a_powertrain_generated.dbc -opendbc/gm_global_a_object.dbc -opendbc/gm_global_a_chassis.dbc - -opendbc/FORD_CADS.dbc -opendbc/ford_fusion_2018_adas.dbc -opendbc/ford_lincoln_base_pt.dbc - -opendbc/honda_accord_2018_can_generated.dbc -opendbc/acura_ilx_2016_can_generated.dbc -opendbc/acura_rdx_2018_can_generated.dbc -opendbc/acura_rdx_2020_can_generated.dbc -opendbc/honda_civic_touring_2016_can_generated.dbc -opendbc/honda_civic_hatchback_ex_2017_can_generated.dbc -opendbc/honda_crv_touring_2016_can_generated.dbc -opendbc/honda_crv_ex_2017_can_generated.dbc -opendbc/honda_crv_ex_2017_body_generated.dbc -opendbc/honda_crv_executive_2016_can_generated.dbc -opendbc/honda_fit_ex_2018_can_generated.dbc -opendbc/honda_odyssey_exl_2018_generated.dbc -opendbc/honda_odyssey_extreme_edition_2018_china_can_generated.dbc -opendbc/honda_insight_ex_2019_can_generated.dbc -opendbc/acura_ilx_2016_nidec.dbc -opendbc/honda_civic_ex_2022_can_generated.dbc - -opendbc/hyundai_canfd.dbc -opendbc/hyundai_kia_generic.dbc -opendbc/hyundai_kia_mando_front_radar_generated.dbc - -opendbc/mazda_2017.dbc - -opendbc/nissan_x_trail_2017_generated.dbc -opendbc/nissan_leaf_2018_generated.dbc - -opendbc/subaru_global_2017_generated.dbc -opendbc/subaru_global_2020_hybrid_generated.dbc -opendbc/subaru_outback_2015_generated.dbc -opendbc/subaru_outback_2019_generated.dbc -opendbc/subaru_forester_2017_generated.dbc - -opendbc/toyota_tnga_k_pt_generated.dbc -opendbc/toyota_new_mc_pt_generated.dbc -opendbc/toyota_nodsu_pt_generated.dbc -opendbc/toyota_adas.dbc -opendbc/toyota_tss2_adas.dbc - -opendbc/vw_golf_mk4.dbc -opendbc/vw_mqb_2010.dbc - -opendbc/tesla_can.dbc -opendbc/tesla_radar_bosch_generated.dbc -opendbc/tesla_radar_continental_generated.dbc -opendbc/tesla_powertrain.dbc - -tinygrad_repo/openpilot/compile2.py -tinygrad_repo/extra/onnx.py -tinygrad_repo/extra/onnx_ops.py -tinygrad_repo/extra/thneed.py -tinygrad_repo/extra/utils.py -tinygrad_repo/tinygrad/codegen/kernel.py -tinygrad_repo/tinygrad/codegen/linearizer.py -tinygrad_repo/tinygrad/features/image.py -tinygrad_repo/tinygrad/features/search.py -tinygrad_repo/tinygrad/nn/* -tinygrad_repo/tinygrad/renderer/cstyle.py -tinygrad_repo/tinygrad/renderer/opencl.py -tinygrad_repo/tinygrad/runtime/lib.py -tinygrad_repo/tinygrad/runtime/ops_cpu.py -tinygrad_repo/tinygrad/runtime/ops_disk.py -tinygrad_repo/tinygrad/runtime/ops_gpu.py -tinygrad_repo/tinygrad/shape/* -tinygrad_repo/tinygrad/*.py diff --git a/release/files_pc b/release/files_pc deleted file mode 100644 index f2bf090f2c..0000000000 --- a/release/files_pc +++ /dev/null @@ -1,4 +0,0 @@ -third_party/libyuv/x86_64/** -third_party/snpe/x86_64/** -third_party/snpe/x86_64-linux-clang/** -third_party/acados/x86_64/** diff --git a/release/files_tici b/release/files_tici deleted file mode 100644 index 18860e20af..0000000000 --- a/release/files_tici +++ /dev/null @@ -1,15 +0,0 @@ -third_party/libyuv/larch64/** -third_party/snpe/larch64** -third_party/snpe/aarch64-ubuntu-gcc7.5/* -third_party/acados/larch64/** - -system/camerad/cameras/camera_qcom2.cc -system/camerad/cameras/camera_qcom2.h -system/camerad/cameras/camera_util.cc -system/camerad/cameras/camera_util.h -system/camerad/cameras/process_raw.cl - -system/qcomgpsd/* - -selfdrive/ui/qt/spinner_larch64 -selfdrive/ui/qt/text_larch64 diff --git a/release/release_files.py b/release/release_files.py new file mode 100755 index 0000000000..bf6cd56a90 --- /dev/null +++ b/release/release_files.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +import os +import re +from pathlib import Path + +HERE = os.path.abspath(os.path.dirname(__file__)) +ROOT = HERE + "/.." + +# blacklisting is for two purposes: +# - minimizing release download size +# - keeping the diff readable +blacklist = [ + "^scripts/", + "body/STL/", + "tools/cabana/", + "panda/examples/", + "opendbc/generator/", + + "^tools/", + "^tinygrad_repo/", + + "matlab.*.md", + + ".git$", # for submodules + ".git/", + ".github/", + ".devcontainer/", + "Darwin/", + ".vscode", +] + +# gets you through the blacklist +whitelist = [ + "tools/lib/", + + "tinygrad_repo/openpilot/compile2.py", + "tinygrad_repo/extra/onnx.py", + "tinygrad_repo/extra/onnx_ops.py", + "tinygrad_repo/extra/thneed.py", + "tinygrad_repo/extra/utils.py", + "tinygrad_repo/tinygrad/codegen/kernel.py", + "tinygrad_repo/tinygrad/codegen/linearizer.py", + "tinygrad_repo/tinygrad/features/image.py", + "tinygrad_repo/tinygrad/features/search.py", + "tinygrad_repo/tinygrad/nn/*", + "tinygrad_repo/tinygrad/renderer/cstyle.py", + "tinygrad_repo/tinygrad/renderer/opencl.py", + "tinygrad_repo/tinygrad/runtime/lib.py", + "tinygrad_repo/tinygrad/runtime/ops_cpu.py", + "tinygrad_repo/tinygrad/runtime/ops_disk.py", + "tinygrad_repo/tinygrad/runtime/ops_gpu.py", + "tinygrad_repo/tinygrad/shape/*", + "tinygrad_repo/tinygrad/*.py", +] + +if __name__ == "__main__": + for f in Path(ROOT).rglob("**/*"): + if not (f.is_file() or f.is_symlink()): + continue + + rf = str(f.relative_to(ROOT)) + blacklisted = any(re.search(p, rf) for p in blacklist) + whitelisted = any(re.search(p, rf) for p in whitelist) + if blacklisted and not whitelisted: + continue + + print(rf) diff --git a/selfdrive/modeld/SConscript b/selfdrive/modeld/SConscript index 2d15223d1e..deb84d5952 100644 --- a/selfdrive/modeld/SConscript +++ b/selfdrive/modeld/SConscript @@ -1,3 +1,5 @@ +import glob + Import('env', 'envCython', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc', 'transformations') lenv = env.Clone() lenvCython = envCython.Clone() @@ -50,11 +52,12 @@ lenvCython.Program('runners/runmodel_pyx.so', 'runners/runmodel_pyx.pyx', LIBS=c lenvCython.Program('runners/snpemodel_pyx.so', 'runners/snpemodel_pyx.pyx', LIBS=[snpemodel_lib, snpe_lib, *cython_libs], FRAMEWORKS=frameworks, RPATH=snpe_rpath) lenvCython.Program('models/commonmodel_pyx.so', 'models/commonmodel_pyx.pyx', LIBS=[commonmodel_lib, *cython_libs], FRAMEWORKS=frameworks) +tinygrad_files = ["#"+x for x in glob.glob(env.Dir("#tinygrad_repo").relpath + "/**", recursive=True, root_dir=env.Dir("#").abspath)] + # Get model metadata fn = File("models/supercombo").abspath cmd = f'python3 {Dir("#selfdrive/modeld").abspath}/get_model_metadata.py {fn}.onnx' -files = sum([lenv.Glob("#"+x) for x in open(File("#release/files_common").abspath).read().split("\n") if x.endswith("get_model_metadata.py")], []) -lenv.Command(fn + "_metadata.pkl", [fn + ".onnx"]+files, cmd) +lenv.Command(fn + "_metadata.pkl", [fn + ".onnx"] + tinygrad_files, cmd) # Build thneed model if arch == "larch64" or GetOption('pc_thneed'): @@ -64,7 +67,6 @@ if arch == "larch64" or GetOption('pc_thneed'): tinygrad_opts += ["FLOAT16=1", "PYOPENCL_NO_CACHE=1"] cmd = f"cd {Dir('#').abspath}/tinygrad_repo && " + ' '.join(tinygrad_opts) + f" python3 openpilot/compile2.py {fn}.onnx {fn}.thneed" - tinygrad_files = sum([lenv.Glob("#"+x) for x in open(File("#release/files_common").abspath).read().split("\n") if x.startswith("tinygrad_repo/")], []) lenv.Command(fn + ".thneed", [fn + ".onnx"] + tinygrad_files, cmd) thneed_lib = env.SharedLibrary('thneed', thneed_src, LIBS=[gpucommon, common, 'zmq', 'OpenCL', 'dl'])