Release cleanup (#1649)

* start cleanup

* whitelist submodule release files

* don't need to ship tests

* revert changes to build script

* add those tests back

* fix typo

* whitelist dbcs

* test

* run the right test

* whitelist opendbc/can

* revert that

* add missing dotfiles

* clean up

* ford DBCs

Co-authored-by: Comma Device <device@comma.ai>

.github/workflows/test.yaml

@@ -28,9 +28,6 @@ jobs:
         cp -pR --parents $(cat release/files_common) $TEST_DIR
         cp Dockerfile.openpilot $TEST_DIR
 
-        # copy submodules
-        cp -pR panda/ opendbc/ cereal/ $TEST_DIR
-
         # need this to build on x86
         cp -pR --parents phonelibs/libyuv phonelibs/snpe \
                          external/bin selfdrive/modeld/runners $TEST_DIR
@@ -41,8 +38,11 @@ jobs:
         mkdir laika laika_repo tools release
     - name: Build Docker image
       run: cd $TEST_DIR && eval "$BUILD"
-    - name: Build openpilot
+    - name: Build openpiot and run quick check
-      run: $RUN "cd /tmp/openpilot && scons -j$(nproc)"
+      run: |
+          $RUN "cd /tmp/openpilot && \
+                scons -j$(nproc) && \
+                $UNIT_TEST selfdrive/car"
 
   docker_push:
     name: docker push

release/build_devel.sh

@@ -9,23 +9,6 @@ echo $$ > /dev/cpuset/app/tasks
 echo $PPID > /dev/cpuset/app/tasks
 
 
-add_subtree() {
-  echo "[-] adding $2 subtree T=$SECONDS"
-  if [ -d "$2" ]; then
-    if git subtree pull --prefix "$2" https://github.com/commaai/"$1".git "$3" --squash -m "Merge $2 subtree"; then
-      echo "git subtree pull succeeds"
-    else
-      echo "git subtree pull failed, fixing"
-      git merge --abort || true
-      git rm -r $2
-      git commit -m "Remove old $2 subtree"
-      git subtree add --prefix "$2" https://github.com/commaai/"$1".git "$3" --squash
-    fi
-  else
-    git subtree add --prefix "$2" https://github.com/commaai/"$1".git "$3" --squash
-  fi
-}
-
 SOURCE_DIR=/data/openpilot_source
 TARGET_DIR=/data/openpilot
 
@@ -39,13 +22,12 @@ export GIT_SSH_COMMAND="ssh -i /tmp/deploy_key"
 
 echo "[-] Setting up repo T=$SECONDS"
 if [ ! -d "$TARGET_DIR" ]; then
-    mkdir -p $TARGET_DIR
+  mkdir -p $TARGET_DIR
-    cd $TARGET_DIR
+  cd $TARGET_DIR
-    git init
+  git init
-    git remote add origin git@github.com:commaai/openpilot.git
+  git remote add origin git@github.com:commaai/openpilot.git
 fi
 
-
 echo "[-] fetching public T=$SECONDS"
 cd $TARGET_DIR
 git prune || true
@@ -61,11 +43,6 @@ git checkout master-ci
 git reset --hard origin/devel
 git clean -xdf
 
-# subtrees to make updates more reliable. updating them needs a clean tree
-add_subtree "cereal" "cereal" master
-add_subtree "panda" "panda" master
-add_subtree "opendbc" "opendbc" master
-
 # leave .git alone
 echo "[-] erasing old openpilot T=$SECONDS"
 rm -rf $TARGET_DIR/* $TARGET_DIR/.gitmodules
@@ -73,9 +50,6 @@ rm -rf $TARGET_DIR/* $TARGET_DIR/.gitmodules
 # delete dotfiles in root
 find . -maxdepth 1 -type f -delete
 
-# dont delete our subtrees
-git checkout -- cereal panda opendbc
-
 # reset tree and get version
 cd $SOURCE_DIR
 git clean -xdf
@@ -122,8 +96,8 @@ make obj/comma.bin
 popd
 
 if [ ! -z "$PUSH" ]; then
-    echo "[-] Pushing to $PUSH T=$SECONDS"
+  echo "[-] Pushing to $PUSH T=$SECONDS"
-    git push -f origin master-ci:$PUSH
+  git push -f origin master-ci:$PUSH
 fi
 
 echo "[-] done pushing T=$SECONDS"

release/files_common

@@ -1,9 +1,6 @@
 README.md
 SAFETY.md
 
-codecov.yml
-lgtm.yml
-
 .gitignore
 LICENSE
 launch_chffrplus.sh
@@ -43,7 +40,6 @@ common/manager_helpers.py
 
 common/kalman/.gitignore
 common/kalman/*
-common/kalman/tests/*
 
 common/transformations/__init__.py
 common/transformations/camera.py
@@ -73,8 +69,6 @@ selfdrive/updated.py
 selfdrive/athena/__init__.py
 selfdrive/athena/athenad.py
 selfdrive/athena/manage_athenad.py
-selfdrive/athena/test.py
-selfdrive/athena/test_helpers.py
 
 selfdrive/boardd/.gitignore
 selfdrive/boardd/SConscript
@@ -84,7 +78,6 @@ selfdrive/boardd/boardd.py
 selfdrive/boardd/boardd_api_impl.pyx
 selfdrive/boardd/boardd_setup.py
 selfdrive/boardd/can_list_to_can_capnp.cc
-selfdrive/boardd/tests/**
 
 selfdrive/car/__init__.py
 selfdrive/car/car_helpers.py
@@ -93,6 +86,8 @@ selfdrive/car/interfaces.py
 selfdrive/car/vin.py
 selfdrive/car/fw_versions.py
 selfdrive/car/isotp_parallel_query.py
+selfdrive/car/tests/__init__.py
+selfdrive/car/tests/test_car_interfaces.py
 selfdrive/car/chrysler/__init__.py
 selfdrive/car/chrysler/carstate.py
 selfdrive/car/chrysler/interface.py
@@ -100,7 +95,6 @@ selfdrive/car/chrysler/radar_interface.py
 selfdrive/car/chrysler/values.py
 selfdrive/car/chrysler/carcontroller.py
 selfdrive/car/chrysler/chryslercan.py
-selfdrive/car/chrysler/test_chryslercan.py
 selfdrive/car/honda/__init__.py
 selfdrive/car/honda/carstate.py
 selfdrive/car/honda/interface.py
@@ -170,7 +164,6 @@ selfdrive/clocksd/.gitignore
 selfdrive/clocksd/SConscript
 selfdrive/clocksd/clocksd.cc
 
-selfdrive/debug/mpc/*
 selfdrive/debug/*.py
 
 selfdrive/common/SConscript
@@ -208,7 +201,6 @@ selfdrive/common/spinner.h
 
 
 selfdrive/controls/__init__.py
-selfdrive/controls/tests/*
 selfdrive/controls/controlsd.py
 selfdrive/controls/plannerd.py
 selfdrive/controls/radard.py
@@ -269,7 +261,6 @@ selfdrive/locationd/ubloxd_main.cc
 selfdrive/locationd/ubloxd_test.cc
 selfdrive/locationd/ublox_msg.cc
 selfdrive/locationd/ublox_msg.h
-selfdrive/locationd/test/*.py
 
 selfdrive/locationd/locationd.py
 selfdrive/locationd/paramsd.py
@@ -308,7 +299,6 @@ selfdrive/loggerd/__init__.py
 selfdrive/loggerd/config.py
 selfdrive/loggerd/uploader.py
 selfdrive/loggerd/deleter.py
-selfdrive/loggerd/tests/*
 
 selfdrive/sensord/SConscript
 selfdrive/sensord/gpsd.cc
@@ -321,19 +311,10 @@ selfdrive/thermald/thermald.py
 selfdrive/thermald/power_monitoring.py
 
 selfdrive/test/__init__.py
-selfdrive/test/longitudinal_maneuvers/*.py
 selfdrive/test/test_openpilot.py
 selfdrive/test/test_fingerprints.py
 selfdrive/test/test_car_models.py
 
-selfdrive/test/process_replay/__init__.py
-selfdrive/test/process_replay/compare_logs.py
-selfdrive/test/process_replay/process_replay.py
-selfdrive/test/process_replay/test_processes.py
-selfdrive/test/process_replay/update_refs.py
-selfdrive/test/process_replay/ref_commit
-selfdrive/test/process_replay/README.md
-
 selfdrive/ui/SConscript
 selfdrive/ui/*.c
 selfdrive/ui/*.cc
@@ -437,6 +418,123 @@ installer/updater/Makefile
 scripts/update_now.sh
 scripts/stop_updater.sh
 
-pyextra/**
+pyextra/logentries/**
 
 rednose/**
+
+cereal/.gitignore
+cereal/__init__.py
+cereal/car.capnp
+cereal/log.capnp
+cereal/services.py
+cereal/service_list.yaml
+cereal/SConscript
+cereal/include/**
+cereal/messaging/.gitignore
+cereal/messaging/__init__.py
+cereal/messaging/bridge.cc
+cereal/messaging/impl_msgq.cc
+cereal/messaging/impl_msgq.hpp
+cereal/messaging/impl_zmq.cc
+cereal/messaging/impl_zmq.hpp
+cereal/messaging/messaging.cc
+cereal/messaging/messaging.hpp
+cereal/messaging/messaging.pxd
+cereal/messaging/messaging_pyx.pyx
+cereal/messaging/messaging_pyx_setup.py
+cereal/messaging/msgq.cc
+cereal/messaging/msgq.hpp
+cereal/messaging/socketmaster.cc
+
+panda/.gitignore
+panda/__init__.py
+panda/VERSION
+panda/board/**
+panda/certs/**
+panda/common/**
+panda/crypto/**
+panda/python/**
+
+opendbc/.gitignore
+opendbc/__init__.py
+opendbc/can/__init__.py
+opendbc/can/SConscript
+opendbc/can/can_define.py
+opendbc/can/common.cc
+opendbc/can/common.h
+opendbc/can/common.pxd
+opendbc/can/common_dbc.h
+opendbc/can/dbc.cc
+opendbc/can/dbc.py
+opendbc/can/dbc_template.cc
+opendbc/can/packer.cc
+opendbc/can/packer.py
+opendbc/can/packer_pyx.pyx
+opendbc/can/packer_pyx_setup.py
+opendbc/can/parser.cc
+opendbc/can/parser.py
+opendbc/can/parser_pyx.pyx
+opendbc/can/parser_pyx_setup.py
+opendbc/can/process_dbc.py
+opendbc/can/dbc_out/.gitkeep
+opendbc/can/dbc_out/.gitignore
+
+opendbc/chrysler_pacifica_2017_hybrid.dbc
+opendbc/chrysler_pacifica_2017_hybrid_private_fusion.dbc
+
+opendbc/gm_global_a_powertrain.dbc
+opendbc/gm_global_a_object.dbc
+opendbc/gm_global_a_chassis.dbc
+
+opendbc/ford_fusion_2018_pt.dbc
+opendbc/ford_fusion_2018_adas.dbc
+
+opendbc/honda_accord_s2t_2018_can_generated.dbc
+opendbc/honda_accord_lx15t_2018_can_generated.dbc
+opendbc/acura_ilx_2016_can_generated.dbc
+opendbc/acura_rdx_2018_can_generated.dbc
+opendbc/honda_civic_touring_2016_can_generated.dbc
+opendbc/honda_civic_hatchback_ex_2017_can_generated.dbc
+opendbc/honda_civic_sedan_16_diesel_2019_can_generated.dbc
+opendbc/honda_crv_touring_2016_can_generated.dbc
+opendbc/honda_crv_ex_2017_can_generated.dbc
+opendbc/honda_crv_executive_2016_can_generated.dbc
+opendbc/honda_crv_hybrid_2019_can_generated.dbc
+opendbc/honda_fit_ex_2018_can_generated.dbc
+opendbc/honda_hrv_touring_2019_can_generated.dbc
+opendbc/honda_odyssey_exl_2018_generated.dbc
+opendbc/honda_odyssey_extreme_edition_2018_china_can_generated.dbc
+opendbc/honda_pilot_touring_2017_can_generated.dbc
+opendbc/honda_ridgeline_black_edition_2017_can_generated.dbc
+opendbc/honda_insight_ex_2019_can_generated.dbc
+opendbc/acura_ilx_2016_nidec.dbc
+
+opendbc/hyundai_kia_generic.dbc
+
+opendbc/mazda_2017.dbc
+
+opendbc/nissan_x_trail_2017.dbc
+opendbc/nissan_leaf_2018.dbc
+
+opendbc/subaru_global_2017.dbc
+
+opendbc/toyota_rav4_hybrid_2017_pt_generated.dbc
+opendbc/toyota_rav4_2017_pt_generated.dbc
+opendbc/toyota_prius_2017_pt_generated.dbc
+opendbc/toyota_corolla_2017_pt_generated.dbc
+opendbc/lexus_rx_350_2016_pt_generated.dbc
+opendbc/lexus_rx_hybrid_2017_pt_generated.dbc
+opendbc/toyota_nodsu_pt_generated.dbc
+opendbc/toyota_nodsu_hybrid_pt_generated.dbc
+opendbc/toyota_camry_hybrid_2018_pt_generated.dbc
+opendbc/toyota_highlander_2017_pt_generated.dbc
+opendbc/toyota_highlander_hybrid_2018_pt_generated.dbc
+opendbc/toyota_avalon_2017_pt_generated.dbc
+opendbc/toyota_sienna_xle_2018_pt_generated.dbc
+opendbc/lexus_is_2018_pt_generated.dbc
+opendbc/lexus_ct200h_2018_pt_generated.dbc
+opendbc/lexus_nx300h_2018_pt_generated.dbc
+opendbc/toyota_adas.dbc
+opendbc/toyota_tss2_adas.dbc
+
+opendbc/vw_mqb_2010.dbc

selfdrive/car/tests/test_car_interfaces.py

@@ -13,6 +13,7 @@ class TestCarInterfaces(unittest.TestCase):
     all_cars = all_known_cars()
 
     for car_name in all_cars:
+      print(car_name)
       fingerprint = FINGERPRINTS[car_name][0]
 
       CarInterface, CarController, CarState = interfaces[car_name]