Make release files a blacklist (#32540)

* blacklist

* little nicer

* whitelist

* cleanup

* igore body

* more skip

.github/workflows/selfdrive_tests.yaml

@@ -53,7 +53,7 @@ jobs:
       run: |
         cd $STRIPPED_DIR
         ${{ env.RUN }} "release/check-dirty.sh && \
-                        MAX_EXAMPLES=5 $PYTEST selfdrive/car"
+                        MAX_EXAMPLES=5 $PYTEST -m 'not slow' selfdrive/car"
     - name: pre-commit
       timeout-minutes: 3
       run: |
@@ -62,7 +62,7 @@ jobs:
         cp pyproject.toml $STRIPPED_DIR
         cp poetry.lock $STRIPPED_DIR
         cd $STRIPPED_DIR
-        ${{ env.RUN }} "unset PYTHONWARNINGS && SKIP=check-added-large-files pre-commit run --all && chmod -R 777 /tmp/pre-commit"
+        ${{ env.RUN }} "unset PYTHONWARNINGS && SKIP=check-added-large-files,check-hooks-apply,check-useless-excludes pre-commit run --all && chmod -R 777 /tmp/pre-commit"
 
   build:
     strategy:

.pre-commit-config.yaml

@@ -47,7 +47,7 @@ repos:
         args:
         - --local-partial-types
         - --explicit-package-bases
-        exclude: '^(third_party/)|(cereal/)|(opendbc/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)'
+        exclude: '^(third_party/)|(body/)|(cereal/)|(opendbc/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)'
 -   repo: local
     hooks:
     -   id: cppcheck

release/build_devel.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/bash
-
 set -ex
 
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
@@ -44,11 +43,12 @@ git clean -xdff
 # do the files copy
 echo "[-] copying files T=$SECONDS"
 cd $SOURCE_DIR
-cp -pR --parents $(cat release/files_*) $TARGET_DIR/
+cp -pR --parents $(./release/release_files.py) $TARGET_DIR/
 
 # in the directory
 cd $TARGET_DIR
 rm -f panda/board/obj/panda.bin.signed
+git submodule status
 
 # include source commit hash and build date in commit
 GIT_HASH=$(git --git-dir=$SOURCE_DIR/.git rev-parse HEAD)

release/build_release.sh

@@ -9,13 +9,6 @@ cd $DIR
 BUILD_DIR=/data/openpilot
 SOURCE_DIR="$(git rev-parse --show-toplevel)"
 
-if [ -f /TICI ]; then
-  FILES_SRC="release/files_tici"
-else
-  echo "no release files set"
-  exit 1
-fi
-
 if [ -z "$RELEASE_BRANCH" ]; then
   echo "RELEASE_BRANCH is not set"
   exit 1
@@ -36,8 +29,7 @@ git checkout --orphan $RELEASE_BRANCH
 # do the files copy
 echo "[-] copying files T=$SECONDS"
 cd $SOURCE_DIR
-cp -pR --parents $(cat release/files_common) $BUILD_DIR/
+cp -pR --parents $(./release/release_files.py) $TARGET_DIR/
-cp -pR --parents $(cat $FILES_SRC) $BUILD_DIR/
 
 # in the directory
 cd $BUILD_DIR
@@ -77,6 +69,10 @@ find . -name '__pycache__' -delete
 rm -rf .sconsign.dblite Jenkinsfile release/
 rm selfdrive/modeld/models/supercombo.onnx
 
+find third_party/ -name '*x86*' -exec rm -r {} +
+find third_party/ -name '*Darwin*' -exec rm -r {} +
+
+
 # Restore third_party
 git checkout third_party/
 

release/files_common

@@ -1,560 +0,0 @@
-.gitignore
-LICENSE
-launch_env.sh
-launch_chffrplus.sh
-launch_openpilot.sh
-
-Jenkinsfile
-SConstruct
-pyproject.toml
-
-README.md
-RELEASES.md
-docs/CARS.md
-docs/CONTRIBUTING.md
-docs/INTEGRATION.md
-docs/LIMITATIONS.md
-site_scons/site_tools/cython.py
-
-openpilot/__init__.py
-openpilot/**
-
-common/.gitignore
-common/__init__.py
-common/*.py
-common/*.pyx
-common/mock/*
-
-common/transformations/__init__.py
-common/transformations/camera.py
-common/transformations/model.py
-
-common/transformations/SConscript
-common/transformations/coordinates.py
-common/transformations/coordinates.cc
-common/transformations/coordinates.hpp
-common/transformations/orientation.py
-common/transformations/orientation.cc
-common/transformations/orientation.hpp
-common/transformations/transformations.pxd
-common/transformations/transformations.pyx
-
-common/api/__init__.py
-
-release/*
-
-tools/__init__.py
-tools/lib/*
-tools/bodyteleop/.gitignore
-tools/bodyteleop/web.py
-tools/bodyteleop/static/*
-tools/joystick/*
-tools/replay/*.cc
-tools/replay/*.h
-
-selfdrive/__init__.py
-
-system/logmessaged.py
-system/micd.py
-system/version.py
-
-selfdrive/SConscript
-
-system/athena/__init__.py
-system/athena/athenad.py
-system/athena/manage_athenad.py
-system/athena/registration.py
-
-selfdrive/boardd/.gitignore
-selfdrive/boardd/SConscript
-selfdrive/boardd/__init__.py
-selfdrive/boardd/boardd.cc
-selfdrive/boardd/boardd.h
-selfdrive/boardd/main.cc
-selfdrive/boardd/boardd.py
-selfdrive/boardd/boardd_api_impl.pyx
-selfdrive/boardd/can_list_to_can_capnp.cc
-selfdrive/boardd/panda.cc
-selfdrive/boardd/panda.h
-selfdrive/boardd/spi.cc
-selfdrive/boardd/panda_comms.h
-selfdrive/boardd/panda_comms.cc
-selfdrive/boardd/pandad.py
-selfdrive/boardd/tests/test_boardd_loopback.py
-
-selfdrive/car/__init__.py
-selfdrive/car/card.py
-selfdrive/car/docs_definitions.py
-selfdrive/car/car_helpers.py
-selfdrive/car/fingerprints.py
-selfdrive/car/interfaces.py
-selfdrive/car/values.py
-selfdrive/car/vin.py
-selfdrive/car/disable_ecu.py
-selfdrive/car/fw_versions.py
-selfdrive/car/fw_query_definitions.py
-selfdrive/car/ecu_addrs.py
-selfdrive/car/isotp_parallel_query.py
-selfdrive/car/tests/__init__.py
-selfdrive/car/tests/test_car_interfaces.py
-selfdrive/car/torque_data/*
-
-selfdrive/car/body/*.py
-selfdrive/car/chrysler/*.py
-selfdrive/car/ford/*.py
-selfdrive/car/gm/*.py
-selfdrive/car/honda/*.py
-selfdrive/car/hyundai/*.py
-selfdrive/car/mazda/*.py
-selfdrive/car/mock/*.py
-selfdrive/car/nissan/*.py
-selfdrive/car/subaru/*.py
-selfdrive/car/tesla/*.py
-selfdrive/car/toyota/*.py
-selfdrive/car/volkswagen/*.py
-
-selfdrive/debug/can_printer.py
-selfdrive/debug/check_freq.py
-selfdrive/debug/dump.py
-selfdrive/debug/filter_log_message.py
-selfdrive/debug/format_fingerprints.py
-selfdrive/debug/get_fingerprint.py
-selfdrive/debug/uiview.py
-
-selfdrive/debug/hyundai_enable_radar_points.py
-selfdrive/debug/vw_mqb_config.py
-
-common/SConscript
-common/version.h
-
-common/*.h
-common/*.cc
-
-selfdrive/controls/__init__.py
-selfdrive/controls/controlsd.py
-selfdrive/controls/plannerd.py
-selfdrive/controls/radard.py
-selfdrive/controls/lib/__init__.py
-selfdrive/controls/lib/alertmanager.py
-selfdrive/controls/lib/alerts_offroad.json
-selfdrive/controls/lib/desire_helper.py
-selfdrive/controls/lib/drive_helpers.py
-selfdrive/controls/lib/events.py
-selfdrive/controls/lib/latcontrol_angle.py
-selfdrive/controls/lib/latcontrol_torque.py
-selfdrive/controls/lib/latcontrol_pid.py
-selfdrive/controls/lib/latcontrol.py
-selfdrive/controls/lib/longcontrol.py
-selfdrive/controls/lib/longitudinal_planner.py
-selfdrive/controls/lib/pid.py
-selfdrive/controls/lib/vehicle_model.py
-
-selfdrive/controls/lib/lateral_mpc_lib/.gitignore
-selfdrive/controls/lib/longitudinal_mpc_lib/.gitignore
-selfdrive/controls/lib/lateral_mpc_lib/*
-selfdrive/controls/lib/longitudinal_mpc_lib/*
-
-system/__init__.py
-system/*.py
-
-system/sentry.py
-system/tombstoned.py
-system/statsd.py
-
-system/hardware/__init__.py
-system/hardware/base.h
-system/hardware/base.py
-system/hardware/hw.h
-system/hardware/hw.py
-system/hardware/tici/__init__.py
-system/hardware/tici/hardware.h
-system/hardware/tici/hardware.py
-system/hardware/tici/pins.py
-system/hardware/tici/agnos.py
-system/hardware/tici/agnos.json
-system/hardware/tici/amplifier.py
-system/hardware/tici/updater
-system/hardware/tici/iwlist.py
-system/hardware/tici/esim.nmconnection
-system/hardware/pc/__init__.py
-system/hardware/pc/hardware.h
-system/hardware/pc/hardware.py
-
-system/ubloxd/.gitignore
-system/ubloxd/SConscript
-system/ubloxd/pigeond.py
-system/ubloxd/generated/*
-system/ubloxd/*.h
-system/ubloxd/*.cc
-
-system/updated/**
-
-selfdrive/locationd/__init__.py
-selfdrive/locationd/SConscript
-selfdrive/locationd/.gitignore
-selfdrive/locationd/locationd.h
-selfdrive/locationd/locationd.cc
-selfdrive/locationd/paramsd.py
-selfdrive/locationd/models/__init__.py
-selfdrive/locationd/models/.gitignore
-selfdrive/locationd/models/car_kf.py
-selfdrive/locationd/models/live_kf.py
-selfdrive/locationd/models/live_kf.h
-selfdrive/locationd/models/live_kf.cc
-selfdrive/locationd/models/constants.py
-
-selfdrive/locationd/torqued.py
-selfdrive/locationd/calibrationd.py
-selfdrive/locationd/helpers.py
-
-system/logcatd/.gitignore
-system/logcatd/SConscript
-system/logcatd/logcatd_systemd.cc
-
-system/proclogd/SConscript
-system/proclogd/main.cc
-system/proclogd/proclog.cc
-system/proclogd/proclog.h
-
-system/loggerd/.gitignore
-system/loggerd/SConscript
-system/loggerd/encoder/encoder.cc
-system/loggerd/encoder/encoder.h
-system/loggerd/encoder/v4l_encoder.cc
-system/loggerd/encoder/v4l_encoder.h
-system/loggerd/video_writer.cc
-system/loggerd/video_writer.h
-system/loggerd/logger.cc
-system/loggerd/logger.h
-system/loggerd/loggerd.cc
-system/loggerd/loggerd.h
-system/loggerd/encoderd.cc
-system/loggerd/bootlog.cc
-system/loggerd/encoder/ffmpeg_encoder.cc
-system/loggerd/encoder/ffmpeg_encoder.h
-
-system/loggerd/__init__.py
-system/loggerd/config.py
-system/loggerd/uploader.py
-system/loggerd/deleter.py
-system/loggerd/xattr_cache.py
-
-system/sensord/.gitignore
-system/sensord/SConscript
-system/sensord/sensors_qcom2.cc
-system/sensord/sensors/*.cc
-system/sensord/sensors/*.h
-
-system/webrtc/__init__.py
-system/webrtc/webrtcd.py
-system/webrtc/schema.py
-system/webrtc/device/audio.py
-system/webrtc/device/video.py
-
-system/thermald/thermald.py
-system/thermald/power_monitoring.py
-system/thermald/fan_controller.py
-
-selfdrive/test/__init__.py
-selfdrive/test/fuzzy_generation.py
-selfdrive/test/helpers.py
-selfdrive/test/setup_device_ci.sh
-selfdrive/test/test_onroad.py
-selfdrive/test/test_time_to_onroad.py
-
-selfdrive/ui/.gitignore
-selfdrive/ui/SConscript
-selfdrive/ui/*.cc
-selfdrive/ui/*.h
-selfdrive/ui/text
-selfdrive/ui/spinner
-selfdrive/ui/soundd.py
-selfdrive/ui/translations/*.ts
-selfdrive/ui/translations/languages.json
-selfdrive/ui/update_translations.py
-selfdrive/ui/tests/test_translations.py
-
-selfdrive/ui/qt/*.cc
-selfdrive/ui/qt/*.h
-selfdrive/ui/qt/network/*.cc
-selfdrive/ui/qt/network/*.h
-selfdrive/ui/qt/offroad/*.cc
-selfdrive/ui/qt/offroad/*.h
-selfdrive/ui/qt/offroad/*.qml
-selfdrive/ui/qt/onroad/*.cc
-selfdrive/ui/qt/onroad/*.h
-selfdrive/ui/qt/widgets/*.cc
-selfdrive/ui/qt/widgets/*.h
-selfdrive/ui/qt/maps/*.cc
-selfdrive/ui/qt/maps/*.h
-selfdrive/ui/qt/setup/*.cc
-selfdrive/ui/qt/setup/*.h
-
-selfdrive/ui/installer/*.cc
-selfdrive/ui/installer/*.h
-selfdrive/ui/installer/*.cc
-
-system/camerad/SConscript
-system/camerad/main.cc
-
-system/camerad/snapshot/*
-system/camerad/cameras/camera_common.h
-system/camerad/cameras/camera_common.cc
-system/camerad/sensors/*.h
-system/camerad/sensors/*.cc
-
-system/manager/__init__.py
-system/manager/**
-
-selfdrive/modeld/.gitignore
-selfdrive/modeld/__init__.py
-selfdrive/modeld/SConscript
-selfdrive/modeld/modeld.py
-selfdrive/modeld/parse_model_outputs.py
-selfdrive/modeld/fill_model_msg.py
-selfdrive/modeld/get_model_metadata.py
-selfdrive/modeld/dmonitoringmodeld.py
-selfdrive/modeld/constants.py
-selfdrive/modeld/modeld
-
-selfdrive/modeld/models/__init__.py
-selfdrive/modeld/models/*.pxd
-selfdrive/modeld/models/*.pyx
-
-selfdrive/modeld/models/commonmodel.cc
-selfdrive/modeld/models/commonmodel.h
-
-selfdrive/modeld/models/supercombo.onnx
-
-selfdrive/modeld/models/dmonitoring_model_q.dlc
-
-selfdrive/modeld/transforms/loadyuv.cc
-selfdrive/modeld/transforms/loadyuv.h
-selfdrive/modeld/transforms/loadyuv.cl
-selfdrive/modeld/transforms/transform.cc
-selfdrive/modeld/transforms/transform.h
-selfdrive/modeld/transforms/transform.cl
-
-selfdrive/modeld/thneed/*.py
-selfdrive/modeld/thneed/thneed.h
-selfdrive/modeld/thneed/thneed_common.cc
-selfdrive/modeld/thneed/thneed_qcom2.cc
-selfdrive/modeld/thneed/serialize.cc
-
-selfdrive/modeld/runners/__init__.py
-selfdrive/modeld/runners/*.pxd
-selfdrive/modeld/runners/*.pyx
-selfdrive/modeld/runners/*.cc
-selfdrive/modeld/runners/*.h
-selfdrive/modeld/runners/*.py
-
-selfdrive/monitoring/dmonitoringd.py
-selfdrive/monitoring/helpers.py
-
-selfdrive/navd/.gitignore
-selfdrive/navd/__init__.py
-selfdrive/navd/**
-
-selfdrive/assets/.gitignore
-selfdrive/assets/assets.qrc
-selfdrive/assets/*.png
-selfdrive/assets/*.svg
-selfdrive/assets/body/*
-selfdrive/assets/fonts/*.ttf
-selfdrive/assets/icons/*
-selfdrive/assets/images/*
-selfdrive/assets/offroad/*
-selfdrive/assets/sounds/*
-selfdrive/assets/training/*
-selfdrive/assets/navigation/*
-
-third_party/.gitignore
-third_party/SConscript
-
-third_party/linux/**
-third_party/opencl/**
-
-third_party/json11/json11.cpp
-third_party/json11/json11.hpp
-
-third_party/qrcode/*.cc
-third_party/qrcode/*.hpp
-
-third_party/kaitai/*.h
-third_party/kaitai/*.cpp
-
-third_party/libyuv/include/**
-
-third_party/snpe/include/**
-third_party/snpe/dsp**
-
-third_party/acados/.gitignore
-third_party/acados/include/**
-third_party/acados/acados_template/**
-
-third_party/bootstrap/**
-third_party/qt5/larch64/bin/**
-third_party/maplibre-native-qt/**
-
-scripts/update_now.sh
-scripts/stop_updater.sh
-
-teleoprtc/**
-
-rednose_repo/site_scons/site_tools/rednose_filter.py
-rednose/.gitignore
-rednose/**
-
-body/.gitignore
-body/board/SConscript
-body/board/*.h
-body/board/*.c
-body/board/*.s
-body/board/*.ld
-body/board/inc/**
-body/board/obj/
-body/board/bldc/**
-body/board/drivers/**
-body/certs/**
-body/crypto/**
-
-cereal/.gitignore
-cereal/__init__.py
-cereal/car.capnp
-cereal/custom.capnp
-cereal/legacy.capnp
-cereal/log.capnp
-cereal/services.py
-cereal/SConscript
-cereal/include/**
-cereal/logger/logger.h
-cereal/messaging/.gitignore
-cereal/messaging/__init__.py
-cereal/messaging/bridge.cc
-cereal/messaging/event.cc
-cereal/messaging/event.h
-cereal/messaging/impl_fake.cc
-cereal/messaging/impl_fake.h
-cereal/messaging/impl_msgq.cc
-cereal/messaging/impl_msgq.h
-cereal/messaging/impl_zmq.cc
-cereal/messaging/impl_zmq.h
-cereal/messaging/messaging.cc
-cereal/messaging/messaging.h
-cereal/messaging/messaging.pxd
-cereal/messaging/messaging_pyx.pyx
-cereal/messaging/msgq.cc
-cereal/messaging/msgq.h
-cereal/messaging/socketmaster.cc
-cereal/visionipc/.gitignore
-cereal/visionipc/__init__.py
-cereal/visionipc/*.cc
-cereal/visionipc/*.h
-cereal/visionipc/*.pyx
-cereal/visionipc/*.pxd
-
-panda/.gitignore
-panda/__init__.py
-panda/SConscript
-panda/board/**
-panda/certs/**
-panda/crypto/**
-panda/examples/query_fw_versions.py
-panda/python/**
-
-opendbc/.gitignore
-opendbc/__init__.py
-opendbc/can/__init__.py
-opendbc/can/SConscript
-opendbc/can/can_define.py
-opendbc/can/common.cc
-opendbc/can/common.h
-opendbc/can/common.pxd
-opendbc/can/common_dbc.h
-opendbc/can/dbc.cc
-opendbc/can/packer.cc
-opendbc/can/packer.py
-opendbc/can/packer_pyx.pyx
-opendbc/can/parser.cc
-opendbc/can/parser.py
-opendbc/can/parser_pyx.pyx
-
-opendbc/comma_body.dbc
-
-opendbc/chrysler_ram_hd_generated.dbc
-opendbc/chrysler_ram_dt_generated.dbc
-opendbc/chrysler_pacifica_2017_hybrid_generated.dbc
-opendbc/chrysler_pacifica_2017_hybrid_private_fusion.dbc
-
-opendbc/gm_global_a_powertrain_generated.dbc
-opendbc/gm_global_a_object.dbc
-opendbc/gm_global_a_chassis.dbc
-
-opendbc/FORD_CADS.dbc
-opendbc/ford_fusion_2018_adas.dbc
-opendbc/ford_lincoln_base_pt.dbc
-
-opendbc/honda_accord_2018_can_generated.dbc
-opendbc/acura_ilx_2016_can_generated.dbc
-opendbc/acura_rdx_2018_can_generated.dbc
-opendbc/acura_rdx_2020_can_generated.dbc
-opendbc/honda_civic_touring_2016_can_generated.dbc
-opendbc/honda_civic_hatchback_ex_2017_can_generated.dbc
-opendbc/honda_crv_touring_2016_can_generated.dbc
-opendbc/honda_crv_ex_2017_can_generated.dbc
-opendbc/honda_crv_ex_2017_body_generated.dbc
-opendbc/honda_crv_executive_2016_can_generated.dbc
-opendbc/honda_fit_ex_2018_can_generated.dbc
-opendbc/honda_odyssey_exl_2018_generated.dbc
-opendbc/honda_odyssey_extreme_edition_2018_china_can_generated.dbc
-opendbc/honda_insight_ex_2019_can_generated.dbc
-opendbc/acura_ilx_2016_nidec.dbc
-opendbc/honda_civic_ex_2022_can_generated.dbc
-
-opendbc/hyundai_canfd.dbc
-opendbc/hyundai_kia_generic.dbc
-opendbc/hyundai_kia_mando_front_radar_generated.dbc
-
-opendbc/mazda_2017.dbc
-
-opendbc/nissan_x_trail_2017_generated.dbc
-opendbc/nissan_leaf_2018_generated.dbc
-
-opendbc/subaru_global_2017_generated.dbc
-opendbc/subaru_global_2020_hybrid_generated.dbc
-opendbc/subaru_outback_2015_generated.dbc
-opendbc/subaru_outback_2019_generated.dbc
-opendbc/subaru_forester_2017_generated.dbc
-
-opendbc/toyota_tnga_k_pt_generated.dbc
-opendbc/toyota_new_mc_pt_generated.dbc
-opendbc/toyota_nodsu_pt_generated.dbc
-opendbc/toyota_adas.dbc
-opendbc/toyota_tss2_adas.dbc
-
-opendbc/vw_golf_mk4.dbc
-opendbc/vw_mqb_2010.dbc
-
-opendbc/tesla_can.dbc
-opendbc/tesla_radar_bosch_generated.dbc
-opendbc/tesla_radar_continental_generated.dbc
-opendbc/tesla_powertrain.dbc
-
-tinygrad_repo/openpilot/compile2.py
-tinygrad_repo/extra/onnx.py
-tinygrad_repo/extra/onnx_ops.py
-tinygrad_repo/extra/thneed.py
-tinygrad_repo/extra/utils.py
-tinygrad_repo/tinygrad/codegen/kernel.py
-tinygrad_repo/tinygrad/codegen/linearizer.py
-tinygrad_repo/tinygrad/features/image.py
-tinygrad_repo/tinygrad/features/search.py
-tinygrad_repo/tinygrad/nn/*
-tinygrad_repo/tinygrad/renderer/cstyle.py
-tinygrad_repo/tinygrad/renderer/opencl.py
-tinygrad_repo/tinygrad/runtime/lib.py
-tinygrad_repo/tinygrad/runtime/ops_cpu.py
-tinygrad_repo/tinygrad/runtime/ops_disk.py
-tinygrad_repo/tinygrad/runtime/ops_gpu.py
-tinygrad_repo/tinygrad/shape/*
-tinygrad_repo/tinygrad/*.py

release/files_pc

@@ -1,4 +0,0 @@
-third_party/libyuv/x86_64/**
-third_party/snpe/x86_64/**
-third_party/snpe/x86_64-linux-clang/**
-third_party/acados/x86_64/**

release/files_tici

@@ -1,15 +0,0 @@
-third_party/libyuv/larch64/**
-third_party/snpe/larch64**
-third_party/snpe/aarch64-ubuntu-gcc7.5/*
-third_party/acados/larch64/**
-
-system/camerad/cameras/camera_qcom2.cc
-system/camerad/cameras/camera_qcom2.h
-system/camerad/cameras/camera_util.cc
-system/camerad/cameras/camera_util.h
-system/camerad/cameras/process_raw.cl
-
-system/qcomgpsd/*
-
-selfdrive/ui/qt/spinner_larch64
-selfdrive/ui/qt/text_larch64

release/release_files.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+import os
+import re
+from pathlib import Path
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+ROOT = HERE + "/.."
+
+# blacklisting is for two purposes:
+# - minimizing release download size
+# - keeping the diff readable
+blacklist = [
+  "^scripts/",
+  "body/STL/",
+  "tools/cabana/",
+  "panda/examples/",
+  "opendbc/generator/",
+
+  "^tools/",
+  "^tinygrad_repo/",
+
+  "matlab.*.md",
+
+  ".git$",  # for submodules
+  ".git/",
+  ".github/",
+  ".devcontainer/",
+  "Darwin/",
+  ".vscode",
+]
+
+# gets you through the blacklist
+whitelist = [
+  "tools/lib/",
+
+  "tinygrad_repo/openpilot/compile2.py",
+  "tinygrad_repo/extra/onnx.py",
+  "tinygrad_repo/extra/onnx_ops.py",
+  "tinygrad_repo/extra/thneed.py",
+  "tinygrad_repo/extra/utils.py",
+  "tinygrad_repo/tinygrad/codegen/kernel.py",
+  "tinygrad_repo/tinygrad/codegen/linearizer.py",
+  "tinygrad_repo/tinygrad/features/image.py",
+  "tinygrad_repo/tinygrad/features/search.py",
+  "tinygrad_repo/tinygrad/nn/*",
+  "tinygrad_repo/tinygrad/renderer/cstyle.py",
+  "tinygrad_repo/tinygrad/renderer/opencl.py",
+  "tinygrad_repo/tinygrad/runtime/lib.py",
+  "tinygrad_repo/tinygrad/runtime/ops_cpu.py",
+  "tinygrad_repo/tinygrad/runtime/ops_disk.py",
+  "tinygrad_repo/tinygrad/runtime/ops_gpu.py",
+  "tinygrad_repo/tinygrad/shape/*",
+  "tinygrad_repo/tinygrad/*.py",
+]
+
+if __name__ == "__main__":
+  for f in Path(ROOT).rglob("**/*"):
+    if not (f.is_file() or f.is_symlink()):
+      continue
+
+    rf = str(f.relative_to(ROOT))
+    blacklisted = any(re.search(p, rf) for p in blacklist)
+    whitelisted = any(re.search(p, rf) for p in whitelist)
+    if blacklisted and not whitelisted:
+      continue
+
+    print(rf)

selfdrive/modeld/SConscript

@@ -1,3 +1,5 @@
+import glob
+
 Import('env', 'envCython', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc', 'transformations')
 lenv = env.Clone()
 lenvCython = envCython.Clone()
@@ -50,11 +52,12 @@ lenvCython.Program('runners/runmodel_pyx.so', 'runners/runmodel_pyx.pyx', LIBS=c
 lenvCython.Program('runners/snpemodel_pyx.so', 'runners/snpemodel_pyx.pyx', LIBS=[snpemodel_lib, snpe_lib, *cython_libs], FRAMEWORKS=frameworks, RPATH=snpe_rpath)
 lenvCython.Program('models/commonmodel_pyx.so', 'models/commonmodel_pyx.pyx', LIBS=[commonmodel_lib, *cython_libs], FRAMEWORKS=frameworks)
 
+tinygrad_files = ["#"+x for x in glob.glob(env.Dir("#tinygrad_repo").relpath + "/**", recursive=True, root_dir=env.Dir("#").abspath)]
+
 # Get model metadata
 fn = File("models/supercombo").abspath
 cmd = f'python3 {Dir("#selfdrive/modeld").abspath}/get_model_metadata.py {fn}.onnx'
-files = sum([lenv.Glob("#"+x) for x in open(File("#release/files_common").abspath).read().split("\n") if x.endswith("get_model_metadata.py")], [])
+lenv.Command(fn + "_metadata.pkl", [fn + ".onnx"] + tinygrad_files, cmd)
-lenv.Command(fn + "_metadata.pkl", [fn + ".onnx"]+files, cmd)
 
 # Build thneed model
 if arch == "larch64" or GetOption('pc_thneed'):
@@ -64,7 +67,6 @@ if arch == "larch64" or GetOption('pc_thneed'):
     tinygrad_opts += ["FLOAT16=1", "PYOPENCL_NO_CACHE=1"]
   cmd = f"cd {Dir('#').abspath}/tinygrad_repo && " + ' '.join(tinygrad_opts) + f" python3 openpilot/compile2.py {fn}.onnx {fn}.thneed"
 
-  tinygrad_files = sum([lenv.Glob("#"+x) for x in open(File("#release/files_common").abspath).read().split("\n") if x.startswith("tinygrad_repo/")], [])
   lenv.Command(fn + ".thneed", [fn + ".onnx"] + tinygrad_files, cmd)
 
   thneed_lib = env.SharedLibrary('thneed', thneed_src, LIBS=[gpucommon, common, 'zmq', 'OpenCL', 'dl'])