CI speedup (#26096)

* CI speedup * use the new stuff * push * no regressions * try that * don't let this slip * fix modeld tests * fix linter * modernize prebuilt * cleanup * fix those * increase a bit
3 years ago · 00494a44f4
parent d109dda720
commit 00494a44f4
16 changed files with 95 additions and 81 deletions
--- a/.github/workflows/prebuilt.yaml
+++ b/.github/workflows/prebuilt.yaml
@ -2,7 +2,6 @@ name: prebuilt
 on:
  schedule:
    - cron: '0 * * * *'
-
  workflow_dispatch:

 env:
@ -11,9 +10,7 @@ env:

  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
  BUILD: |
-      docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) || true
-      docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
-      docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
+      DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .

 jobs:
  build_prebuilt:
@ -37,8 +34,7 @@ jobs:
    - name: Build Docker image
      run: |
        eval "$BUILD"
-        docker pull $DOCKER_REGISTRY/$IMAGE_NAME:latest || true
-        docker build --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f Dockerfile.openpilot .
+        DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f Dockerfile.openpilot .
    - name: Push to container registry
      run: |
        $DOCKER_LOGIN
--- a/.github/workflows/selfdrive_tests.yaml
+++ b/.github/workflows/selfdrive_tests.yaml
@ -18,15 +18,12 @@ env:

  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
  BUILD: |
-      docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) || true
-      docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
-      docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
+      DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .

  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c

  BUILD_CL: |
-      docker pull $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest || true
-      docker build --cache-from $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $CL_BASE_IMAGE:latest -f Dockerfile.openpilot_base_cl .
+      DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $CL_BASE_IMAGE:latest -f Dockerfile.openpilot_base_cl .
  RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c

  UNIT_TEST: coverage run --append -m unittest discover
@ -41,13 +38,10 @@ jobs:
    steps:
    - uses: actions/checkout@v3
      with:
-        fetch-depth: 0
        submodules: true
    - name: Build devel
      run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
    - uses: ./.github/workflows/setup
-      with:
-        save-cache: true
    - name: Check submodules
      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
      run: release/check-submodules.sh
@ -72,18 +66,20 @@ jobs:
  build_all:
    name: build all
    runs-on: ubuntu-20.04
-    timeout-minutes: 50
+    timeout-minutes: 30
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - uses: ./.github/workflows/setup
+      with:
+        save-cache: true
    - name: Build openpilot with all flags
      run: ${{ env.RUN }} "scons -j$(nproc) --extras && release/check-dirty.sh"
    - name: Cleanup scons cache
      run: |
          ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
-                          scons -j$(nproc) --extras --cache-populate"
+                          scons -j$(nproc) --cache-populate"

  #build_mac:
  #  name: build macos
@ -145,7 +141,7 @@ jobs:
  docker_push:
    name: docker push
    runs-on: ubuntu-20.04
-    timeout-minutes: 50
+    timeout-minutes: 22
    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
    needs: static_analysis # hack to ensure slow tests run first since this and static_analysis are fast
    steps:
@ -154,12 +150,14 @@ jobs:
        submodules: true
    - name: Build Docker image
      run: eval "$BUILD"
+      timeout-minutes: 13
    - name: Push to container registry
      run: |
        $DOCKER_LOGIN
        docker push $DOCKER_REGISTRY/$BASE_IMAGE:latest
    - name: Build CL Docker image
      run: eval "$BUILD_CL"
+      timeout-minutes: 4
    - name: Push to container registry
      run: |
        $DOCKER_LOGIN
@ -168,7 +166,7 @@ jobs:
  static_analysis:
    name: static analysis
    runs-on: ubuntu-20.04
-    timeout-minutes: 50
+    timeout-minutes: 20
    steps:
    - uses: actions/checkout@v3
      with:
@ -176,21 +174,23 @@ jobs:
    - name: Build Docker image
      run: eval "$BUILD"
    - name: pre-commit
+      timeout-minutes: 5
      run: ${{ env.RUN }} "pre-commit run --all"

  valgrind:
    name: valgrind
    runs-on: ubuntu-20.04
-    timeout-minutes: 50
+    timeout-minutes: 20
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - uses: ./.github/workflows/setup
+    - name: Build openpilot
+      run: ${{ env.RUN }} "scons -j$(nproc)"
    - name: Run valgrind
      run: |
-        ${{ env.RUN }} "scons -j$(nproc) && \
-                        python selfdrive/test/test_valgrind_replay.py"
+        ${{ env.RUN }} "python selfdrive/test/test_valgrind_replay.py"
    - name: Print logs
      if: always()
      run: cat selfdrive/test/valgrind_logs.txt
@ -198,16 +198,18 @@ jobs:
  unit_tests:
    name: unit tests
    runs-on: ubuntu-20.04
-    timeout-minutes: 50
+    timeout-minutes: 30
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - uses: ./.github/workflows/setup
+    - name: Build openpilot
+      run: ${{ env.RUN }} "scons -j$(nproc)"
    - name: Run unit tests
+      timeout-minutes: 15
      run: |
        ${{ env.RUN }} "export SKIP_LONG_TESTS=1 && \
-                        scons -j$(nproc) && \
                        $UNIT_TEST common && \
                        $UNIT_TEST opendbc/can && \
                        $UNIT_TEST selfdrive/boardd && \
@ -220,7 +222,6 @@ jobs:
                        $UNIT_TEST selfdrive/athena && \
                        $UNIT_TEST selfdrive/thermald && \
                        $UNIT_TEST system/hardware/tici && \
-                        $UNIT_TEST selfdrive/modeld && \
                        $UNIT_TEST tools/lib/tests && \
                        ./selfdrive/ui/tests/create_test_translations.sh && \
                        QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
@ -239,7 +240,7 @@ jobs:
  process_replay:
    name: process replay
    runs-on: ubuntu-20.04
-    timeout-minutes: 50
+    timeout-minutes: 25
    steps:
    - uses: actions/checkout@v3
      with:
@ -251,10 +252,12 @@ jobs:
      with:
        path: /tmp/comma_download_cache
        key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit') }}
+    - name: Build openpilot
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
    - name: Run replay
      run: |
-        ${{ env.RUN }} "scons -j$(nproc) && \
-                        CI=1 coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
+        ${{ env.RUN }} "CI=1 coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
                        coverage xml"
    - name: Print diff
      if: always()
@ -268,15 +271,14 @@ jobs:
    - name: Upload reference logs
      if: ${{ failure() && github.event_name == 'pull_request' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
      run: |
-        ${{ env.RUN }} "scons -j$(nproc) && \
-                        CI=1 AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
+        ${{ env.RUN }} "CI=1 AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v2

-  model_replay_onnx:
-    name: model replay onnx
+  test_modeld:
+    name: model tests
    runs-on: ubuntu-20.04
-    timeout-minutes: 50
+    timeout-minutes: 20
    steps:
    - uses: actions/checkout@v3
      with:
@ -285,29 +287,41 @@ jobs:
    - name: Build Docker image
      # Sim docker is needed to get the OpenCL drivers
      run: eval "$BUILD_CL"
-    - name: Run replay
+    - name: Build openpilot
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+    - name: Run model replay with ONNX
+      timeout-minutes: 2
+      run: |
+        ${{ env.RUN_CL }} "ONNXCPU=1 CI=1 coverage run selfdrive/test/process_replay/model_replay.py && \
+                           coverage xml"
+    - name: Run unit tests
+      timeout-minutes: 5
      run: |
-        ${{ env.RUN_CL }} "scons -j$(nproc) && \
-                           ONNXCPU=1 CI=1 coverage run \
-                           selfdrive/test/process_replay/model_replay.py -j$(nproc) && \
+        ${{ env.RUN_CL }} "$UNIT_TEST selfdrive/modeld && \
                           coverage xml"
+    - name: "Upload coverage to Codecov"
+      uses: codecov/codecov-action@v2

  test_longitudinal:
    name: longitudinal
    runs-on: ubuntu-20.04
-    timeout-minutes: 50
+    timeout-minutes: 20
    steps:
    - uses: actions/checkout@v3
      with:
        submodules: true
    - uses: ./.github/workflows/setup
+    - name: Build openpilot
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
    - name: Test longitudinal
      run: |
        ${{ env.RUN }} "mkdir -p selfdrive/test/out && \
-                        scons -j$(nproc) && \
                        cd selfdrive/test/longitudinal_maneuvers && \
                        coverage run ./test_longitudinal.py && \
                        coverage xml"
+      timeout-minutes: 2
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v2
    - uses: actions/upload-artifact@v2
@ -320,7 +334,7 @@ jobs:
  test_cars:
    name: cars
    runs-on: ubuntu-20.04
-    timeout-minutes: 50
+    timeout-minutes: 20
    strategy:
      fail-fast: false
      matrix:
@ -336,10 +350,12 @@ jobs:
      with:
        path: /tmp/comma_download_cache
        key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }}
+    - name: Build openpilot
+      run: ${{ env.RUN }} "scons -j$(nproc)"
    - name: Test car models
+      timeout-minutes: 12
      run: |
-        ${{ env.RUN }} "scons -j$(nproc) && \
-                        coverage run -m pytest selfdrive/car/tests/test_models.py && \
+        ${{ env.RUN }} "coverage run -m pytest selfdrive/car/tests/test_models.py && \
                        coverage xml && \
                        chmod -R 777 /tmp/comma_download_cache"
      env:
@ -348,29 +364,10 @@ jobs:
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v2

-  docs:
-    name: build docs
-    runs-on: ubuntu-20.04
-    timeout-minutes: 50
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: true
-    - name: Build docker container
-      run: |
-        docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
-        docker pull $DOCKER_REGISTRY/openpilot-docs:latest || true
-        DOCKER_BUILDKIT=1 docker build --cache-from $DOCKER_REGISTRY/openpilot-docs:latest -t $DOCKER_REGISTRY/openpilot-docs:latest -f docs/docker/Dockerfile .
-    - name: Push docker container
-      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
-      run: |
-        $DOCKER_LOGIN
-        docker push $DOCKER_REGISTRY/openpilot-docs:latest
-
  car_docs_diff:
-    name: comment on PR with car docs diff
+    name: PR comments
    runs-on: ubuntu-20.04
-    timeout-minutes: 50
+    timeout-minutes: 20
    if: github.event_name == 'pull_request'
    steps:
      - uses: actions/checkout@v3
--- a/.github/workflows/setup/action.yaml
+++ b/.github/workflows/setup/action.yaml
@ -15,9 +15,9 @@ runs:
    # build cache
    - id: date
      shell: bash
-      run: echo "::set-output name=date::$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d')"
+      run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
    - shell: bash
-      run: echo "${{ steps.date.outputs.date }}"
+      run: echo "$CACHE_COMMIT_DATE"
    - shell: bash
      run: echo "CACHE_SKIP_SAVE=true" >> $GITHUB_ENV
      if: github.ref != 'refs/heads/master' || inputs.save-cache == 'false'
@ -27,9 +27,9 @@ runs:
      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
      with:
        path: /tmp/scons_cache
-        key: scons-${{ steps.date.outputs.date }}-${{ github.sha }}
+        key: scons-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
        restore-keys: |
-          scons-${{ steps.date.outputs.date }}-
+          scons-${{ env.CACHE_COMMIT_DATE }}-
          scons-

    # build our docker image
--- a/.github/workflows/tools_tests.yaml
+++ b/.github/workflows/tools_tests.yaml
@ -2,6 +2,8 @@ name: tools

 on:
  push:
+    branches-ignore:
+      - 'testing-closet*'
  pull_request:

 concurrency:
@ -15,21 +17,20 @@ env:
  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}

  BUILD: |
-      docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) || true
-      docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
-      docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
+      DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
+
+  RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
+
  BUILD_CL: |
-      docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base_cl) || true
-      docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
-      docker build --cache-from $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $CL_BASE_IMAGE:latest -f Dockerfile.openpilot_base_cl .
-  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e \
-       GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/comma_download_cache:/tmp/comma_download_cache $BASE_IMAGE /bin/sh -c
+      DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $CL_BASE_IMAGE:latest -f Dockerfile.openpilot_base_cl .
+  RUN_CL: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
+

 jobs:
  plotjuggler:
    name: plotjuggler
    runs-on: ubuntu-20.04
-    timeout-minutes: 30
+    timeout-minutes: 20
    steps:
    - uses: actions/checkout@v3
      with:
@ -37,6 +38,7 @@ jobs:
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Unit test
+      timeout-minutes: 2
      run: |
        ${{ env.RUN }} "scons -j$(nproc) --directory=/tmp/openpilot/cereal && \
                        apt-get update && \
@ -47,7 +49,7 @@ jobs:
  simulator:
    name: simulator
    runs-on: ubuntu-20.04
-    timeout-minutes: 50
+    timeout-minutes: 30
    env:
      IMAGE_NAME: openpilot-sim
    if: github.repository == 'commaai/openpilot'
@ -61,12 +63,29 @@ jobs:
      run: eval "$BUILD"
    - name: Build base cl image
      run: eval "$BUILD_CL"
-    - name: Pull latest simulator image
-      run: docker pull $DOCKER_REGISTRY/$IMAGE_NAME:latest || true
    - name: Build simulator image
-      run: docker build --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/sim/Dockerfile.sim .
+      run: DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/sim/Dockerfile.sim .
    - name: Push to container registry
      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
      run: |
        $DOCKER_LOGIN
        docker push $DOCKER_REGISTRY/$IMAGE_NAME:latest
+
+  docs:
+    name: build docs
+    runs-on: ubuntu-20.04
+    timeout-minutes: 25
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+    - name: Build docker container
+      run: |
+        DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/openpilot-docs:latest -t $DOCKER_REGISTRY/openpilot-docs:latest -f docs/docker/Dockerfile .
+    - name: Push docker container
+      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
+      run: |
+        $DOCKER_LOGIN
+        docker push $DOCKER_REGISTRY/openpilot-docs:latest
+
+
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@ -53,6 +53,7 @@ repos:
        types: [python]
        exclude: '^(pyextra/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(laika_repo/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)'
        args:
+        - -j0
        - -rn
        - -sn
        - --rcfile=.pylintrc
--- a/release/build_devel.sh
+++ b/release/build_devel.sh
@ -22,8 +22,8 @@ pre-commit uninstall || true

 echo "[-] bringing master-ci and devel in sync T=$SECONDS"
 cd $TARGET_DIR
-git fetch origin master-ci
-git fetch origin devel
+git fetch --depth 1 origin master-ci
+git fetch --depth 1 origin devel

 git checkout -f --track origin/master-ci
 git reset --hard master-ci
--- a/selfdrive/modeld/tests/init.py
+++ b/selfdrive/modeld/tests/init.py
--- a/selfdrive/modeld/tests/dmon_lag/repro.cc
+++ b/selfdrive/modeld/tests/dmon_lag/repro.cc
--- a/selfdrive/modeld/tests/snpe_benchmark/.gitignore
+++ b/selfdrive/modeld/tests/snpe_benchmark/.gitignore
--- a/selfdrive/modeld/tests/snpe_benchmark/benchmark.cc
+++ b/selfdrive/modeld/tests/snpe_benchmark/benchmark.cc
@ -102,6 +102,7 @@ void get_testframe(int index, std::unique_ptr<zdl::DlSystem::ITensor> &input) {
  fread(frame_buffer, length, 1, pFile);
  // std::cout << *(frame_buffer+length/4-1) << std::endl;
  std::copy(frame_buffer, frame_buffer+(length/4), input->begin());
+  fclose(pFile);
 }

 void SaveITensor(const std::string& path, const zdl::DlSystem::ITensor* tensor)
--- a/selfdrive/modeld/tests/snpe_benchmark/benchmark.sh
+++ b/selfdrive/modeld/tests/snpe_benchmark/benchmark.sh
--- a/selfdrive/modeld/tests/test_modeld.py
+++ b/selfdrive/modeld/tests/test_modeld.py
--- a/selfdrive/modeld/tests/tf_test/build.sh
+++ b/selfdrive/modeld/tests/tf_test/build.sh
--- a/selfdrive/modeld/tests/tf_test/main.cc
+++ b/selfdrive/modeld/tests/tf_test/main.cc
--- a/selfdrive/modeld/tests/tf_test/pb_loader.py
+++ b/selfdrive/modeld/tests/tf_test/pb_loader.py
--- a/selfdrive/modeld/tests/timing/benchmark.py
+++ b/selfdrive/modeld/tests/timing/benchmark.py