Integrate codecov.io (#1465)

* integrate codecov

* add codecov to readme

* don't need the token

* fix badges

* /bin/bash, not /bin/sh

* needs coverage in the docker

* from the git root

* add CI_ENV like https://docs.codecov.io/docs/testing-with-docker

* custom upload step

* use codecov action

* coverage xml

* add unit tests, good luck

* name the container tmppilot

* add name, fix unit tests

* yml, fix coveragerc-app

* remove tmp/openpilot

* try without xml

* just call the upload script in docker

* ugh, i hate docker

* idk env

* yaml?

* don't be clever

* maybe this

* running out of ideas

* coverage xml maybe

* upload coverage.xml

* ugh CI_RUN

* unneeded artifact

Co-authored-by: George Hotz <geohot@gmail.com>
old-commit-hash: 4252028a84

.coveragerc-app

@@ -0,0 +1,3 @@
+[run]
+concurrency=multiprocessing
+

.github/workflows/test.yaml

@@ -3,8 +3,10 @@ on: [push, pull_request]
 
 env:
   RUN: docker run --shm-size 1G --rm tmppilot /bin/sh -c
+  PERSIST: docker run --shm-size 1G --name tmppilot tmppilot /bin/sh -c
   LOAD: docker load -i tmppilot.tar.gz/tmppilot.tar.gz
-  UNIT_TEST: cd /tmp/openpilot && python -m unittest discover
+  CI_RUN: docker run -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID --rm tmppilotci /bin/bash -c
+  UNIT_TEST: coverage run --append -m unittest discover
 
 jobs:
   build:
@@ -72,16 +74,21 @@ jobs:
         run: $LOAD
       - name: Run unit tests
         run: |
-          $RUN /tmp/openpilot/selfdrive/test/test_fingerprints.py
+          $PERSIST "cd /tmp/openpilot && \
-          $RUN "$UNIT_TEST common"
+                    coverage run selfdrive/test/test_fingerprints.py && \
-          $RUN "$UNIT_TEST opendbc/can"
+                    $UNIT_TEST common && \
-          $RUN "$UNIT_TEST selfdrive/boardd"
+                    $UNIT_TEST opendbc/can && \
-          $RUN "$UNIT_TEST selfdrive/controls"
+                    $UNIT_TEST selfdrive/boardd && \
-          $RUN "$UNIT_TEST selfdrive/loggerd"
+                    $UNIT_TEST selfdrive/controls && \
-          $RUN "$UNIT_TEST selfdrive/car"
+                    $UNIT_TEST selfdrive/loggerd && \
-          $RUN "$UNIT_TEST selfdrive/locationd"
+                    $UNIT_TEST selfdrive/car && \
-          $RUN "$UNIT_TEST selfdrive/athena"
+                    $UNIT_TEST selfdrive/locationd && \
-          $RUN "$UNIT_TEST tools/lib/tests"
+                    $UNIT_TEST selfdrive/athena && \
+                    $UNIT_TEST tools/lib/tests"
+      - name: Upload coverage to Codecov
+        run: |
+          docker commit tmppilot tmppilotci
+          $CI_RUN "cd /tmp/openpilot && bash <(curl -s https://codecov.io/bash) -Z -F unit_tests"
 
   process_replay:
     name: process replay
@@ -96,14 +103,15 @@ jobs:
         run: $LOAD
       - name: Run replay
         run: |
-          CONTAINER_NAME="tmppilot_${GITHUB_SHA}"
+          $PERSIST "cd /tmp/openpilot && CI=1 coverage run selfdrive/test/process_replay/test_processes.py"
-          docker run --shm-size 1G --name ${CONTAINER_NAME} tmppilot /bin/sh -c "cd /tmp/openpilot/selfdrive/test/process_replay && CI=1 ./test_processes.py"
+      - name: Upload coverage to Codecov
+        run: |
+          docker commit tmppilot tmppilotci
+          $CI_RUN "cd /tmp/openpilot && bash <(curl -s https://codecov.io/bash) -Z -F process_replay"
       - name: Copy diff
         if: always()
         run: |
-          CONTAINER_NAME="tmppilot_${GITHUB_SHA}"
+          docker cp tmppilot:/tmp/openpilot/selfdrive/test/process_replay/diff.txt diff.txt
-          docker cp $CONTAINER_NAME:/tmp/openpilot/selfdrive/test/process_replay/diff.txt diff.txt
-          docker rm $CONTAINER_NAME
       - name: Print diff
         if: always()
         run: cat diff.txt
@@ -126,15 +134,12 @@ jobs:
         run: $LOAD
       - name: Test longitudinal
         run: |
-          CONTAINER_NAME="tmppilot_${GITHUB_SHA}"
+          $PERSIST "cd /tmp/openpilot/selfdrive/test/longitudinal_maneuvers && OPTEST=1 ./test_longitudinal.py"
-          docker run --shm-size 1G --name ${CONTAINER_NAME} tmppilot /bin/sh -c "cd /tmp/openpilot/selfdrive/test/longitudinal_maneuvers && OPTEST=1 ./test_longitudinal.py"
       - name: Copy artifacts
         if: always()
         run: |
-          CONTAINER_NAME="tmppilot_${GITHUB_SHA}"
           mkdir out
-          docker cp $CONTAINER_NAME:/tmp/openpilot/selfdrive/test/longitudinal_maneuvers/out/longitudinal/ out/
+          docker cp tmppilot:/tmp/openpilot/selfdrive/test/longitudinal_maneuvers/out/longitudinal/ out/
-          docker rm $CONTAINER_NAME
       - uses: actions/upload-artifact@v1
         if: always()
         with:
@@ -153,4 +158,10 @@ jobs:
       - name: Load image
         run: $LOAD
       - name: Test car models
-        run: $RUN "mkdir -p /data/params && cd /tmp/openpilot/selfdrive/test/ && ./test_car_models.py"
+        run: |
+          $PERSIST "mkdir -p /data/params && cd /tmp/openpilot && coverage run --parallel-mode --concurrency=multiprocessing --rcfile=./.coveragerc-app selfdrive/test/test_car_models.py && coverage combine"
+      - name: Upload coverage to Codecov
+        run: |
+          docker commit tmppilot tmppilotci
+          $CI_RUN "cd /tmp/openpilot && bash <(curl -s https://codecov.io/bash) -Z -F test_car_models"
+

.gitignore

@@ -56,6 +56,7 @@ xx
 panda_jungle
 
 .coverage*
+coverage.xml
 htmlcov
 pandaextra
 

Dockerfile.openpilot

@@ -61,7 +61,7 @@ RUN python --version
 RUN cd /tmp && pipenv install --system --deploy
 
 # Install subset of dev dependencies needed for CI
-RUN pip install matplotlib==3.1.1 dictdiffer==0.8.0 fastcluster==1.1.25 aenum==2.2.1 lru-dict==1.1.6 scipy==1.4.1 tenacity==5.1.1 azure-common==1.1.23 azure-nspkg==3.0.2 azure-storage-blob==2.1.0 azure-storage-common==2.1.0 azure-storage-nspkg==3.1.0 pycurl==7.43.0.3
+RUN pip install matplotlib==3.1.1 dictdiffer==0.8.0 fastcluster==1.1.25 aenum==2.2.1 lru-dict==1.1.6 scipy==1.4.1 tenacity==5.1.1 azure-common==1.1.23 azure-nspkg==3.0.2 azure-storage-blob==2.1.0 azure-storage-common==2.1.0 azure-storage-nspkg==3.1.0 pycurl==7.43.0.3 coverage==5.1
 
 ENV PATH="/tmp/openpilot/external/bin:${PATH}"
 ENV PYTHONPATH /tmp/openpilot:${PYTHONPATH}
@@ -71,6 +71,8 @@ COPY ./flake8_openpilot.sh /tmp/openpilot/
 COPY ./pylint_openpilot.sh /tmp/openpilot/
 COPY ./.pylintrc /tmp/openpilot/
 
+COPY ./.coveragerc-app /tmp/openpilot/
+
 COPY ./pyextra /tmp/openpilot/pyextra
 COPY ./phonelibs /tmp/openpilot/phonelibs
 COPY ./external /tmp/openpilot/external

README.md

@@ -330,3 +330,5 @@ NO WARRANTY EXPRESSED OR IMPLIED.**
 
 [![Total alerts](https://img.shields.io/lgtm/alerts/g/commaai/openpilot.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/commaai/openpilot/alerts/)
 [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/commaai/openpilot.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/commaai/openpilot/context:python)
+[![Language grade: C/C++](https://img.shields.io/lgtm/grade/cpp/g/commaai/openpilot.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/commaai/openpilot/context:cpp)
+[![codecov](https://codecov.io/gh/commaai/openpilot/branch/master/graph/badge.svg)](https://codecov.io/gh/commaai/openpilot)

external/bin/MP4Box

@@ -1 +0,0 @@
-../gpac/bin/MP4Box

external/bin/apktool

@@ -1 +0,0 @@
-../apktool/apktool

external/bin/ipfs

@@ -1 +0,0 @@
-../ipfs/ipfs