add caching to CI tests (#2269)

* first caching attempt

* caching attempt

* caching attempt

* caching attempt

* caching attempt

* caching attempt

* dump logs

* does this work?

* Create cache folder

* Print folder contents

* Fix volume mount

* Update test.yaml

* add removed tests

* reduce diff

* reduce diff

* reduce diff

* reduce diff

* reduce diff

* readd coverage upload

* Update test.yaml

* add more caching

* Use hash of test file to compute cache key

* add cache to valgrind tests

* Update .github/workflows/test.yaml

Co-authored-by: Willem Melching <willem.melching@gmail.com>

* Update .github/workflows/test.yaml

Co-authored-by: Willem Melching <willem.melching@gmail.com>

* Also hash workflow yaml

Co-authored-by: Willem Melching <willem.melching@gmail.com>

.github/workflows/test.yaml

@@ -6,6 +6,7 @@ on:
 
 env:
   RUN: docker run --shm-size 1G --rm tmppilot /bin/sh -c
+  PERSIST_WITH_CACHE: docker run --shm-size 1G -v /tmp/comma_download_cache:/tmp/comma_download_cache  --name tmppilot tmppilot /bin/sh -c
   PERSIST: docker run --shm-size 1G --name tmppilot tmppilot /bin/sh -c
   CI_RUN: docker run -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID --rm tmppilotci /bin/bash -c
   UNIT_TEST: coverage run --append -m unittest discover
@@ -19,6 +20,7 @@ env:
       docker pull docker.io/commaai/openpilotci:latest || true
       docker build --cache-from docker.io/commaai/openpilotci:latest -t tmppilot -f Dockerfile.openpilotci .
 
+
 jobs:
   build_release:
     name: build release
@@ -38,7 +40,7 @@ jobs:
 
         # need this to build on x86
         cp -pR --parents phonelibs/libyuv phonelibs/snpe \
-                         external/bin selfdrive/modeld/runners $TEST_DIR
+                          external/bin selfdrive/modeld/runners $TEST_DIR
 
         # need these so docker copy won't fail
         cp Pipfile Pipfile.lock .pylintrc .pre-commit-config.yaml $TEST_DIR
@@ -69,7 +71,7 @@ jobs:
           ~/Library/Caches/pip
           ~/Library/Caches/pipenv
           ~/Library/Caches/Homebrew
-        key: ${{ hashFiles('tools/mac_setup.sh') }}
+        key: ${{ hashFiles('.github/workflows/test.yaml', 'tools/mac_setup.sh') }}
     - name: Install dependencies
       run: ./tools/mac_setup.sh
     - name: Build openpilot
@@ -136,13 +138,20 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: true
+      - name: Cache dependencies
+        id: dependency-cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            /tmp/comma_download_cache
+          key: ${{ hashFiles('.github/workflows/test.yaml', 'selfdrive/test/test_valgrind_replay.py') }}
       - name: Build Docker image
         run: eval "$BUILD"
       - name: Run valgrind
         run: |
-          $PERSIST "cd /tmp/openpilot && \
+          $PERSIST_WITH_CACHE "cd /tmp/openpilot && \
                     scons -j$(nproc) && \
-                    python selfdrive/test/test_valgrind_replay.py"
+                    FILEREADER_CACHE=1 python selfdrive/test/test_valgrind_replay.py"
       - name: Print logs
         if: always()
         run: |
@@ -193,13 +202,20 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: true
+      - name: Cache dependencies
+        id: dependency-cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            /tmp/comma_download_cache
+          key: ${{ hashFiles('.github/workflows/test.yaml', 'selfdrive/test/process_replay/test_processes.py') }}
       - name: Build Docker image
         run: eval "$BUILD"
       - name: Run replay
         run: |
-          $PERSIST "cd /tmp/openpilot && \
+          $PERSIST_WITH_CACHE  "cd /tmp/openpilot && \
-                    scons -j$(nproc) && \
+          scons -j$(nproc) && \
-                    CI=1 coverage run selfdrive/test/process_replay/test_processes.py"
+          FILEREADER_CACHE=1 CI=1 coverage run selfdrive/test/process_replay/test_processes.py"
       - name: Upload coverage to Codecov
         run: |
           docker commit tmppilot tmppilotci
@@ -251,13 +267,20 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: true
+      - name: Cache dependencies
+        id: dependency-cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            /tmp/comma_download_cache
+          key: ${{ hashFiles('.github/workflows/test.yaml', 'selfdrive/test/test_car_models.py') }}
       - name: Build Docker image
         run: eval "$BUILD"
       - name: Test car models
         run: |
-          $PERSIST "cd /tmp/openpilot && \
+          $PERSIST_WITH_CACHE "cd /tmp/openpilot && \
                     scons -j$(nproc) && \
-                    coverage run --parallel-mode -m nose --processes=4 --process-timeout=60 \
+                    FILEREADER_CACHE=1 coverage run --parallel-mode -m nose --processes=4 --process-timeout=60 \v
                                  selfdrive/test/test_models.py && \
                     coverage combine"
       - name: Upload coverage to Codecov

selfdrive/test/test_valgrind_replay.py

@@ -98,6 +98,5 @@ class TestValgrind(unittest.TestCase):
       self.replay_process(cfg, lr)
       time.sleep(1)  # Wait for the logs to get written
 
-
 if __name__ == "__main__":
   unittest.main()