From daea932130ce772c5736e47f51d0b72b3340d06e Mon Sep 17 00:00:00 2001 From: Adeeb <8762862+adeebshihadeh@users.noreply.github.com> Date: Mon, 8 Jun 2020 14:24:14 -0700 Subject: [PATCH] Release cleanup (#1649) * start cleanup * whitelist submodule release files * don't need to ship tests * revert changes to build script * add those tests back * fix typo * whitelist dbcs * test * run the right test * whitelist opendbc/can * revert that * add missing dotfiles * clean up * ford DBCs Co-authored-by: Comma Device old-commit-hash: c24a7e248a295641f58d39e0db25dde1b742fc16 --- .github/workflows/test.yaml | 10 +- release/build_devel.sh | 38 +----- release/files_common | 142 +++++++++++++++++---- selfdrive/car/tests/test_car_interfaces.py | 1 + 4 files changed, 132 insertions(+), 59 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ced9fe65cc..d2b04ef1a6 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -28,9 +28,6 @@ jobs: cp -pR --parents $(cat release/files_common) $TEST_DIR cp Dockerfile.openpilot $TEST_DIR - # copy submodules - cp -pR panda/ opendbc/ cereal/ $TEST_DIR - # need this to build on x86 cp -pR --parents phonelibs/libyuv phonelibs/snpe \ external/bin selfdrive/modeld/runners $TEST_DIR @@ -41,8 +38,11 @@ jobs: mkdir laika laika_repo tools release - name: Build Docker image run: cd $TEST_DIR && eval "$BUILD" - - name: Build openpilot - run: $RUN "cd /tmp/openpilot && scons -j$(nproc)" + - name: Build openpiot and run quick check + run: | + $RUN "cd /tmp/openpilot && \ + scons -j$(nproc) && \ + $UNIT_TEST selfdrive/car" docker_push: name: docker push diff --git a/release/build_devel.sh b/release/build_devel.sh index c3028d5520..32c8baf869 100755 --- a/release/build_devel.sh +++ b/release/build_devel.sh @@ -9,23 +9,6 @@ echo $$ > /dev/cpuset/app/tasks echo $PPID > /dev/cpuset/app/tasks -add_subtree() { - echo "[-] adding $2 subtree T=$SECONDS" - if [ -d "$2" ]; then - if git subtree pull --prefix "$2" https://github.com/commaai/"$1".git "$3" --squash -m "Merge $2 subtree"; then - echo "git subtree pull succeeds" - else - echo "git subtree pull failed, fixing" - git merge --abort || true - git rm -r $2 - git commit -m "Remove old $2 subtree" - git subtree add --prefix "$2" https://github.com/commaai/"$1".git "$3" --squash - fi - else - git subtree add --prefix "$2" https://github.com/commaai/"$1".git "$3" --squash - fi -} - SOURCE_DIR=/data/openpilot_source TARGET_DIR=/data/openpilot @@ -39,13 +22,12 @@ export GIT_SSH_COMMAND="ssh -i /tmp/deploy_key" echo "[-] Setting up repo T=$SECONDS" if [ ! -d "$TARGET_DIR" ]; then - mkdir -p $TARGET_DIR - cd $TARGET_DIR - git init - git remote add origin git@github.com:commaai/openpilot.git + mkdir -p $TARGET_DIR + cd $TARGET_DIR + git init + git remote add origin git@github.com:commaai/openpilot.git fi - echo "[-] fetching public T=$SECONDS" cd $TARGET_DIR git prune || true @@ -61,11 +43,6 @@ git checkout master-ci git reset --hard origin/devel git clean -xdf -# subtrees to make updates more reliable. updating them needs a clean tree -add_subtree "cereal" "cereal" master -add_subtree "panda" "panda" master -add_subtree "opendbc" "opendbc" master - # leave .git alone echo "[-] erasing old openpilot T=$SECONDS" rm -rf $TARGET_DIR/* $TARGET_DIR/.gitmodules @@ -73,9 +50,6 @@ rm -rf $TARGET_DIR/* $TARGET_DIR/.gitmodules # delete dotfiles in root find . -maxdepth 1 -type f -delete -# dont delete our subtrees -git checkout -- cereal panda opendbc - # reset tree and get version cd $SOURCE_DIR git clean -xdf @@ -122,8 +96,8 @@ make obj/comma.bin popd if [ ! -z "$PUSH" ]; then - echo "[-] Pushing to $PUSH T=$SECONDS" - git push -f origin master-ci:$PUSH + echo "[-] Pushing to $PUSH T=$SECONDS" + git push -f origin master-ci:$PUSH fi echo "[-] done pushing T=$SECONDS" diff --git a/release/files_common b/release/files_common index a0fc3743ff..1bd4929770 100644 --- a/release/files_common +++ b/release/files_common @@ -1,9 +1,6 @@ README.md SAFETY.md -codecov.yml -lgtm.yml - .gitignore LICENSE launch_chffrplus.sh @@ -43,7 +40,6 @@ common/manager_helpers.py common/kalman/.gitignore common/kalman/* -common/kalman/tests/* common/transformations/__init__.py common/transformations/camera.py @@ -73,8 +69,6 @@ selfdrive/updated.py selfdrive/athena/__init__.py selfdrive/athena/athenad.py selfdrive/athena/manage_athenad.py -selfdrive/athena/test.py -selfdrive/athena/test_helpers.py selfdrive/boardd/.gitignore selfdrive/boardd/SConscript @@ -84,7 +78,6 @@ selfdrive/boardd/boardd.py selfdrive/boardd/boardd_api_impl.pyx selfdrive/boardd/boardd_setup.py selfdrive/boardd/can_list_to_can_capnp.cc -selfdrive/boardd/tests/** selfdrive/car/__init__.py selfdrive/car/car_helpers.py @@ -93,6 +86,8 @@ selfdrive/car/interfaces.py selfdrive/car/vin.py selfdrive/car/fw_versions.py selfdrive/car/isotp_parallel_query.py +selfdrive/car/tests/__init__.py +selfdrive/car/tests/test_car_interfaces.py selfdrive/car/chrysler/__init__.py selfdrive/car/chrysler/carstate.py selfdrive/car/chrysler/interface.py @@ -100,7 +95,6 @@ selfdrive/car/chrysler/radar_interface.py selfdrive/car/chrysler/values.py selfdrive/car/chrysler/carcontroller.py selfdrive/car/chrysler/chryslercan.py -selfdrive/car/chrysler/test_chryslercan.py selfdrive/car/honda/__init__.py selfdrive/car/honda/carstate.py selfdrive/car/honda/interface.py @@ -170,7 +164,6 @@ selfdrive/clocksd/.gitignore selfdrive/clocksd/SConscript selfdrive/clocksd/clocksd.cc -selfdrive/debug/mpc/* selfdrive/debug/*.py selfdrive/common/SConscript @@ -208,7 +201,6 @@ selfdrive/common/spinner.h selfdrive/controls/__init__.py -selfdrive/controls/tests/* selfdrive/controls/controlsd.py selfdrive/controls/plannerd.py selfdrive/controls/radard.py @@ -269,7 +261,6 @@ selfdrive/locationd/ubloxd_main.cc selfdrive/locationd/ubloxd_test.cc selfdrive/locationd/ublox_msg.cc selfdrive/locationd/ublox_msg.h -selfdrive/locationd/test/*.py selfdrive/locationd/locationd.py selfdrive/locationd/paramsd.py @@ -308,7 +299,6 @@ selfdrive/loggerd/__init__.py selfdrive/loggerd/config.py selfdrive/loggerd/uploader.py selfdrive/loggerd/deleter.py -selfdrive/loggerd/tests/* selfdrive/sensord/SConscript selfdrive/sensord/gpsd.cc @@ -321,19 +311,10 @@ selfdrive/thermald/thermald.py selfdrive/thermald/power_monitoring.py selfdrive/test/__init__.py -selfdrive/test/longitudinal_maneuvers/*.py selfdrive/test/test_openpilot.py selfdrive/test/test_fingerprints.py selfdrive/test/test_car_models.py -selfdrive/test/process_replay/__init__.py -selfdrive/test/process_replay/compare_logs.py -selfdrive/test/process_replay/process_replay.py -selfdrive/test/process_replay/test_processes.py -selfdrive/test/process_replay/update_refs.py -selfdrive/test/process_replay/ref_commit -selfdrive/test/process_replay/README.md - selfdrive/ui/SConscript selfdrive/ui/*.c selfdrive/ui/*.cc @@ -437,6 +418,123 @@ installer/updater/Makefile scripts/update_now.sh scripts/stop_updater.sh -pyextra/** +pyextra/logentries/** rednose/** + +cereal/.gitignore +cereal/__init__.py +cereal/car.capnp +cereal/log.capnp +cereal/services.py +cereal/service_list.yaml +cereal/SConscript +cereal/include/** +cereal/messaging/.gitignore +cereal/messaging/__init__.py +cereal/messaging/bridge.cc +cereal/messaging/impl_msgq.cc +cereal/messaging/impl_msgq.hpp +cereal/messaging/impl_zmq.cc +cereal/messaging/impl_zmq.hpp +cereal/messaging/messaging.cc +cereal/messaging/messaging.hpp +cereal/messaging/messaging.pxd +cereal/messaging/messaging_pyx.pyx +cereal/messaging/messaging_pyx_setup.py +cereal/messaging/msgq.cc +cereal/messaging/msgq.hpp +cereal/messaging/socketmaster.cc + +panda/.gitignore +panda/__init__.py +panda/VERSION +panda/board/** +panda/certs/** +panda/common/** +panda/crypto/** +panda/python/** + +opendbc/.gitignore +opendbc/__init__.py +opendbc/can/__init__.py +opendbc/can/SConscript +opendbc/can/can_define.py +opendbc/can/common.cc +opendbc/can/common.h +opendbc/can/common.pxd +opendbc/can/common_dbc.h +opendbc/can/dbc.cc +opendbc/can/dbc.py +opendbc/can/dbc_template.cc +opendbc/can/packer.cc +opendbc/can/packer.py +opendbc/can/packer_pyx.pyx +opendbc/can/packer_pyx_setup.py +opendbc/can/parser.cc +opendbc/can/parser.py +opendbc/can/parser_pyx.pyx +opendbc/can/parser_pyx_setup.py +opendbc/can/process_dbc.py +opendbc/can/dbc_out/.gitkeep +opendbc/can/dbc_out/.gitignore + +opendbc/chrysler_pacifica_2017_hybrid.dbc +opendbc/chrysler_pacifica_2017_hybrid_private_fusion.dbc + +opendbc/gm_global_a_powertrain.dbc +opendbc/gm_global_a_object.dbc +opendbc/gm_global_a_chassis.dbc + +opendbc/ford_fusion_2018_pt.dbc +opendbc/ford_fusion_2018_adas.dbc + +opendbc/honda_accord_s2t_2018_can_generated.dbc +opendbc/honda_accord_lx15t_2018_can_generated.dbc +opendbc/acura_ilx_2016_can_generated.dbc +opendbc/acura_rdx_2018_can_generated.dbc +opendbc/honda_civic_touring_2016_can_generated.dbc +opendbc/honda_civic_hatchback_ex_2017_can_generated.dbc +opendbc/honda_civic_sedan_16_diesel_2019_can_generated.dbc +opendbc/honda_crv_touring_2016_can_generated.dbc +opendbc/honda_crv_ex_2017_can_generated.dbc +opendbc/honda_crv_executive_2016_can_generated.dbc +opendbc/honda_crv_hybrid_2019_can_generated.dbc +opendbc/honda_fit_ex_2018_can_generated.dbc +opendbc/honda_hrv_touring_2019_can_generated.dbc +opendbc/honda_odyssey_exl_2018_generated.dbc +opendbc/honda_odyssey_extreme_edition_2018_china_can_generated.dbc +opendbc/honda_pilot_touring_2017_can_generated.dbc +opendbc/honda_ridgeline_black_edition_2017_can_generated.dbc +opendbc/honda_insight_ex_2019_can_generated.dbc +opendbc/acura_ilx_2016_nidec.dbc + +opendbc/hyundai_kia_generic.dbc + +opendbc/mazda_2017.dbc + +opendbc/nissan_x_trail_2017.dbc +opendbc/nissan_leaf_2018.dbc + +opendbc/subaru_global_2017.dbc + +opendbc/toyota_rav4_hybrid_2017_pt_generated.dbc +opendbc/toyota_rav4_2017_pt_generated.dbc +opendbc/toyota_prius_2017_pt_generated.dbc +opendbc/toyota_corolla_2017_pt_generated.dbc +opendbc/lexus_rx_350_2016_pt_generated.dbc +opendbc/lexus_rx_hybrid_2017_pt_generated.dbc +opendbc/toyota_nodsu_pt_generated.dbc +opendbc/toyota_nodsu_hybrid_pt_generated.dbc +opendbc/toyota_camry_hybrid_2018_pt_generated.dbc +opendbc/toyota_highlander_2017_pt_generated.dbc +opendbc/toyota_highlander_hybrid_2018_pt_generated.dbc +opendbc/toyota_avalon_2017_pt_generated.dbc +opendbc/toyota_sienna_xle_2018_pt_generated.dbc +opendbc/lexus_is_2018_pt_generated.dbc +opendbc/lexus_ct200h_2018_pt_generated.dbc +opendbc/lexus_nx300h_2018_pt_generated.dbc +opendbc/toyota_adas.dbc +opendbc/toyota_tss2_adas.dbc + +opendbc/vw_mqb_2010.dbc diff --git a/selfdrive/car/tests/test_car_interfaces.py b/selfdrive/car/tests/test_car_interfaces.py index c6c908efd8..1800a71d03 100755 --- a/selfdrive/car/tests/test_car_interfaces.py +++ b/selfdrive/car/tests/test_car_interfaces.py @@ -13,6 +13,7 @@ class TestCarInterfaces(unittest.TestCase): all_cars = all_known_cars() for car_name in all_cars: + print(car_name) fingerprint = FINGERPRINTS[car_name][0] CarInterface, CarController, CarState = interfaces[car_name]