sadmen
/
openpilot_comma
mirror of https://github.com/commaai/openpilot.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fast CI (#1537)

Browse Source 
* don't compile in docker build stage

* disable that for now

* do we need that?

* upload artifact already compresses

* that wasn't faster

This reverts commit 2221e13c71.

* try that

* don't think that's used

* this too

* check this

* less layers

* that's not in the dockerfile anymore

* system

* even less layers and clear pipenv cache

* no inline comment i guess

* uninstall pipenv after we're done with it

* separate build job adds too much overhead

* quotes

* clean that up

* increase timeout

* cleanup
pull/1530/head
Adeeb 5 years ago committed by GitHub
parent 8d46c0fa8a
commit 45ee131a44
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 90 additions and 93 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 131
      .github/workflows/test.yaml
  2. 52
      Dockerfile.openpilot

131
.github/workflows/test.yaml
Unescape View File

@ -1,37 +1,21 @@
name: Openpilot Tests name: openpilot tests
on: [push, pull_request] on: [push, pull_request]
env: env:
RUN: docker run --shm-size 1G --rm tmppilot /bin/sh -c RUN: docker run --shm-size 1G --rm tmppilot /bin/sh -c
PERSIST: docker run --shm-size 1G --name tmppilot tmppilot /bin/sh -c PERSIST: docker run --shm-size 1G --name tmppilot tmppilot /bin/sh -c
LOAD: docker load -i tmppilot.tar.gz/tmppilot.tar.gz
CI_RUN: docker run -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID --rm tmppilotci /bin/bash -c CI_RUN: docker run -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID --rm tmppilotci /bin/bash -c
UNIT_TEST: coverage run --append -m unittest discover UNIT_TEST: coverage run --append -m unittest discover
BUILD: |
docker pull $(grep -ioP '(?<=^from)\s+\S+' Dockerfile.openpilot) || true
docker pull docker.io/commaai/openpilot:latest || true
docker build --cache-from docker.io/commaai/openpilot:latest -t tmppilot -f Dockerfile.openpilot .
jobs: jobs:
build:
name: build
runs-on: ubuntu-16.04
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Build docker image
run: |
docker pull $(grep -ioP '(?<=^from)\s+\S+' Dockerfile.openpilot) || true
docker pull docker.io/commaai/openpilot:latest || true
docker build --cache-from docker.io/commaai/openpilot:latest -t tmppilot -f Dockerfile.openpilot .
docker save tmppilot:latest | gzip > tmppilot.tar.gz
- uses: actions/upload-artifact@v2
with:
name: tmppilot.tar.gz
path: tmppilot.tar.gz
build_release: build_release:
name: build release name: build release
runs-on: ubuntu-16.04 runs-on: ubuntu-16.04
timeout-minutes: 30 timeout-minutes: 50
env: env:
TEST_DIR: tmppilot TEST_DIR: tmppilot
steps: steps:
@ -56,41 +40,37 @@ jobs:
.coveragerc-app $TEST_DIR .coveragerc-app $TEST_DIR
cd $TEST_DIR cd $TEST_DIR
mkdir pyextra laika laika_repo tools release mkdir pyextra laika laika_repo tools release
- name: Build - name: Build Docker image
run: | run: cd $TEST_DIR && eval "$BUILD"
cd $TEST_DIR - name: Build openpilot
docker pull $(grep -ioP '(?<=^from)\s+\S+' Dockerfile.openpilot) || true run: $RUN "cd /tmp/openpilot && scons -j$(nproc)"
docker pull docker.io/commaai/openpilot:latest || true
docker build --cache-from docker.io/commaai/openpilot:latest -t tmppilot -f Dockerfile.openpilot .
push: docker_push:
name: push name: docker push
runs-on: ubuntu-16.04 runs-on: ubuntu-16.04
needs: build
if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
needs: linter # hack to ensure slow tests run first since this and linter are fast
steps: steps:
- uses: actions/download-artifact@v1 - uses: actions/checkout@v2
with: with:
name: tmppilot.tar.gz submodules: true
- name: Load image - name: Build Docker image
run: $LOAD run: eval "$BUILD"
- name: Login to dockerhub - name: Push to dockerhub
run: docker login -u wmelching -p ${{ secrets.DOCKERHUB_TOKEN }} run: |
- name: Tag image docker login -u wmelching -p ${{ secrets.DOCKERHUB_TOKEN }}
run: docker tag tmppilot docker.io/commaai/openpilot:latest docker tag tmppilot docker.io/commaai/openpilot:latest
- name: Push image docker push docker.io/commaai/openpilot:latest
run: docker push docker.io/commaai/openpilot:latest
linter: linter:
name: linter name: linter
runs-on: ubuntu-16.04 runs-on: ubuntu-16.04
needs: build
steps: steps:
- uses: actions/download-artifact@v1 - uses: actions/checkout@v2
with: with:
name: tmppilot.tar.gz submodules: true
- name: Load image - name: Build Docker image
run: $LOAD run: eval "$BUILD"
- name: flake8 - name: flake8
run: $RUN "cd /tmp/openpilot/ && ./flake8_openpilot.sh" run: $RUN "cd /tmp/openpilot/ && ./flake8_openpilot.sh"
- name: pylint - name: pylint
@ -99,16 +79,16 @@ jobs:
unit_tests: unit_tests:
name: unit tests name: unit tests
runs-on: ubuntu-16.04 runs-on: ubuntu-16.04
needs: build
steps: steps:
- uses: actions/download-artifact@v1 - uses: actions/checkout@v2
with: with:
name: tmppilot.tar.gz submodules: true
- name: Load image - name: Build Docker image
run: $LOAD run: eval "$BUILD"
- name: Run unit tests - name: Run unit tests
run: | run: |
$PERSIST "cd /tmp/openpilot && \ $PERSIST "cd /tmp/openpilot && \
scons -j$(nproc) && \
coverage run selfdrive/test/test_fingerprints.py && \ coverage run selfdrive/test/test_fingerprints.py && \
$UNIT_TEST common && \ $UNIT_TEST common && \
$UNIT_TEST opendbc/can && \ $UNIT_TEST opendbc/can && \
@ -127,17 +107,18 @@ jobs:
process_replay: process_replay:
name: process replay name: process replay
runs-on: ubuntu-16.04 runs-on: ubuntu-16.04
needs: build timeout-minutes: 50
timeout-minutes: 30
steps: steps:
- uses: actions/download-artifact@v1 - uses: actions/checkout@v2
with: with:
name: tmppilot.tar.gz submodules: true
- name: Load image - name: Build Docker image
run: $LOAD run: eval "$BUILD"
- name: Run replay - name: Run replay
run: | run: |
$PERSIST "cd /tmp/openpilot && CI=1 coverage run selfdrive/test/process_replay/test_processes.py" $PERSIST "cd /tmp/openpilot && \
scons -j$(nproc) && \
CI=1 coverage run selfdrive/test/process_replay/test_processes.py"
- name: Upload coverage to Codecov - name: Upload coverage to Codecov
run: | run: |
docker commit tmppilot tmppilotci docker commit tmppilot tmppilotci
@ -158,17 +139,20 @@ jobs:
test_longitudinal: test_longitudinal:
name: longitudinal name: longitudinal
runs-on: ubuntu-16.04 runs-on: ubuntu-16.04
needs: build timeout-minutes: 50
timeout-minutes: 30
steps: steps:
- uses: actions/download-artifact@v1 - uses: actions/checkout@v2
with: with:
name: tmppilot.tar.gz submodules: true
- name: Load image - name: Build Docker image
run: $LOAD run: eval "$BUILD"
- name: Test longitudinal - name: Test longitudinal
run: | run: |
$PERSIST "cd /tmp/openpilot/selfdrive/test/longitudinal_maneuvers && OPTEST=1 ./test_longitudinal.py" $PERSIST "mkdir -p /tmp/openpilot/selfdrive/test/out && \
cd /tmp/openpilot/ && \
scons -j$(nproc) && \
cd selfdrive/test/longitudinal_maneuvers && \
OPTEST=1 ./test_longitudinal.py"
- name: Copy artifacts - name: Copy artifacts
if: always() if: always()
run: | run: |
@ -183,17 +167,20 @@ jobs:
test_car_models: test_car_models:
name: test car models name: test car models
runs-on: ubuntu-16.04 runs-on: ubuntu-16.04
needs: build timeout-minutes: 50
timeout-minutes: 30
steps: steps:
- uses: actions/download-artifact@v1 - uses: actions/checkout@v2
with: with:
name: tmppilot.tar.gz submodules: true
- name: Load image - name: Build Docker image
run: $LOAD run: eval "$BUILD"
- name: Test car models - name: Test car models
run: | run: |
$PERSIST "mkdir -p /data/params && cd /tmp/openpilot && coverage run --parallel-mode --concurrency=multiprocessing --rcfile=./.coveragerc-app selfdrive/test/test_car_models.py && coverage combine" $PERSIST "mkdir -p /data/params && \
cd /tmp/openpilot && \
scons -j$(nproc) && \
coverage run --parallel-mode --concurrency=multiprocessing --rcfile=./.coveragerc-app selfdrive/test/test_car_models.py && \
coverage combine"
- name: Upload coverage to Codecov - name: Upload coverage to Codecov
run: | run: |
docker commit tmppilot tmppilotci docker commit tmppilot tmppilotci

52
Dockerfile.openpilot
Unescape View File

@ -1,5 +1,6 @@
FROM ubuntu:16.04 FROM ubuntu:16.04
ENV PYTHONUNBUFFERED 1 ENV PYTHONUNBUFFERED 1
ENV PYTHONPATH /tmp/openpilot:${PYTHONPATH}
RUN apt-get update && apt-get install -y --no-install-recommends \ RUN apt-get update && apt-get install -y --no-install-recommends \
autoconf \ autoconf \
@ -22,12 +23,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
libglfw3-dev \ libglfw3-dev \
libglib2.0-0 \ libglib2.0-0 \
liblzma-dev \ liblzma-dev \
libmysqlclient-dev \
libomp-dev \ libomp-dev \
libopencv-dev \ libopencv-dev \
libssl-dev \ libssl-dev \
libsqlite3-dev \ libsqlite3-dev \
libtool \
libusb-1.0-0-dev \ libusb-1.0-0-dev \
libczmq-dev \ libczmq-dev \
libzmq3-dev \ libzmq3-dev \
@ -39,33 +38,49 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
python-pip \ python-pip \
sudo \ sudo \
wget \ wget \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
ENV LANG en_US.UTF-8 ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8 ENV LC_ALL en_US.UTF-8
# Install python dependencies
RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}" ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}"
RUN pyenv install 3.8.2 && pyenv global 3.8.2
RUN pyenv rehash
RUN pip install --no-cache-dir pipenv==2018.11.26
COPY Pipfile Pipfile.lock /tmp/ COPY Pipfile Pipfile.lock /tmp/
RUN pyenv install 3.8.2 && \
RUN cd /tmp && pipenv install --system --deploy pyenv global 3.8.2 && \
pyenv rehash && \
# Install subset of dev dependencies needed for CI pip install --no-cache-dir pipenv==2018.11.26 && \
RUN pip install --no-cache-dir matplotlib==3.1.1 dictdiffer==0.8.0 fastcluster==1.1.25 aenum==2.2.1 lru-dict==1.1.6 scipy==1.4.1 tenacity==5.1.1 azure-common==1.1.23 azure-nspkg==3.0.2 azure-storage-blob==2.1.0 azure-storage-common==2.1.0 azure-storage-nspkg==3.1.0 pycurl==7.43.0.3 coverage==5.1 cd /tmp && \
pipenv install --system --deploy --clear && \
ENV PYTHONPATH /tmp/openpilot:${PYTHONPATH} pip uninstall -y pipenv && \
pip install --no-cache-dir \
matplotlib==3.1.1 \
dictdiffer==0.8.0 \
fastcluster==1.1.25 \
aenum==2.2.1 \
lru-dict==1.1.6 \
scipy==1.4.1 \
tenacity==5.1.1 \
azure-common==1.1.23 \
azure-nspkg==3.0.2 \
azure-storage-blob==2.1.0 \
azure-storage-common==2.1.0 \
azure-storage-nspkg==3.1.0 \
pycurl==7.43.0.3 \
coverage==5.1
RUN mkdir -p /tmp/openpilot RUN mkdir -p /tmp/openpilot
COPY ./flake8_openpilot.sh ./pylint_openpilot.sh ./.pylintrc ./.coveragerc-app /tmp/openpilot/ COPY SConstruct \
flake8_openpilot.sh \
pylint_openpilot.sh \
.pylintrc \
.coveragerc-app \
/tmp/openpilot/
COPY ./pyextra /tmp/openpilot/pyextra COPY ./pyextra /tmp/openpilot/pyextra
COPY ./phonelibs /tmp/openpilot/phonelibs COPY ./phonelibs /tmp/openpilot/phonelibs
@ -79,8 +94,3 @@ COPY ./opendbc /tmp/openpilot/opendbc
COPY ./cereal /tmp/openpilot/cereal COPY ./cereal /tmp/openpilot/cereal
COPY ./panda /tmp/openpilot/panda COPY ./panda /tmp/openpilot/panda
COPY ./selfdrive /tmp/openpilot/selfdrive COPY ./selfdrive /tmp/openpilot/selfdrive
COPY SConstruct /tmp/openpilot/SConstruct
RUN mkdir -p /tmp/openpilot/selfdrive/test/out
RUN cd /tmp/openpilot && scons -j$(nproc)

Write Preview
Loading…
Cancel
Save