From 993b6b43ac172b500c55b8d5b861d1655c8c6620 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sun, 23 Aug 2020 18:18:19 -0700 Subject: [PATCH] split dockerfile into base image and CI image (#2066) * split dockerfile into base image and CI * doesn't exist yet * fix path * cache on this branch * latest * use cache * don't double pull old-commit-hash: a3f56430195d5b2d0119f8bbf929f07c9a625959 --- .dockerignore | 16 +++++++ .github/workflows/test.yaml | 13 ++++-- ...ile.openpilot => Dockerfile.openpilot_base | 42 +------------------ Dockerfile.openpilotci | 24 +++++++++++ Jenkinsfile | 2 +- 5 files changed, 53 insertions(+), 44 deletions(-) rename Dockerfile.openpilot => Dockerfile.openpilot_base (53%) create mode 100644 Dockerfile.openpilotci diff --git a/.dockerignore b/.dockerignore index 2b3e7c54fd..556cc328f0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,21 @@ .git .DS_Store +*.dylib +*.DSYM +*.d +*.pyc +*.pyo +.*.swp +.*.swo +.*.un~ +*.tmp +*.o +*.o-* +*.os +*.os-* +*.so +*.a + notebooks phone massivemap diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7d0ae1f456..7bcaf54cee 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -11,9 +11,14 @@ env: CI_RUN: docker run -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID --rm tmppilotci /bin/bash -c UNIT_TEST: coverage run --append -m unittest discover BUILD: | - docker pull $(grep -ioP '(?<=^from)\s+\S+' Dockerfile.openpilot) || true + # build the openpilot docker base image + docker pull $(grep -ioP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) || true + docker pull docker.io/commaai/openpilot-base:latest || true + docker build --cache-from docker.io/commaai/openpilot-base:latest -t commaai/openpilot-base:latest -f Dockerfile.openpilot_base . + + # build the final CI image docker pull docker.io/commaai/openpilotci:latest || true - docker build --cache-from docker.io/commaai/openpilotci:latest -t tmppilot -f Dockerfile.openpilot . + docker build --cache-from docker.io/commaai/openpilotci:latest -t tmppilot -f Dockerfile.openpilotci . jobs: build_release: @@ -30,7 +35,7 @@ jobs: run: | mkdir $TEST_DIR cp -pR --parents $(cat release/files_common) $TEST_DIR - cp Dockerfile.openpilot $TEST_DIR + cp Dockerfile.openpilot_base Dockerfile.openpilotci $TEST_DIR # need this to build on x86 cp -pR --parents phonelibs/libyuv phonelibs/snpe \ @@ -88,6 +93,8 @@ jobs: - name: Push to dockerhub run: | docker login -u wmelching -p ${{ secrets.COMMA_DOCKERHUB_TOKEN}} + docker tag commaai/openpilot-base:latest docker.io/commaai/openpilot-base:latest + docker push docker.io/commaai/openpilot-base:latest docker tag tmppilot docker.io/commaai/openpilotci:latest docker push docker.io/commaai/openpilotci:latest diff --git a/Dockerfile.openpilot b/Dockerfile.openpilot_base similarity index 53% rename from Dockerfile.openpilot rename to Dockerfile.openpilot_base index 8577282126..06e2a95a09 100644 --- a/Dockerfile.openpilot +++ b/Dockerfile.openpilot_base @@ -1,6 +1,5 @@ FROM ubuntu:16.04 ENV PYTHONUNBUFFERED 1 -ENV PYTHONPATH /tmp/openpilot:${PYTHONPATH} RUN apt-get update && apt-get install -y --no-install-recommends \ autoconf \ @@ -58,42 +57,5 @@ RUN pyenv install 3.8.2 && \ pip install --no-cache-dir --upgrade pip==20.1.1 && \ pip install --no-cache-dir pipenv==2018.11.26 && \ cd /tmp && \ - pipenv install --system --deploy --clear && \ - pip uninstall -y pipenv && \ - pip install --no-cache-dir \ - matplotlib==3.1.1 \ - dictdiffer==0.8.0 \ - fastcluster==1.1.25 \ - aenum==2.2.1 \ - lru-dict==1.1.6 \ - scipy==1.4.1 \ - tenacity==5.1.1 \ - azure-common==1.1.23 \ - azure-nspkg==3.0.2 \ - azure-storage-blob==2.1.0 \ - azure-storage-common==2.1.0 \ - azure-storage-nspkg==3.1.0 \ - pycurl==7.43.0.3 \ - coverage==5.1 \ - pre-commit==2.4.0 \ - parameterized==0.7.4 - -RUN mkdir -p /tmp/openpilot - -COPY SConstruct \ - .pylintrc \ - .pre-commit-config.yaml \ - /tmp/openpilot/ - -COPY ./pyextra /tmp/openpilot/pyextra -COPY ./phonelibs /tmp/openpilot/phonelibs -COPY ./laika /tmp/openpilot/laika -COPY ./laika_repo /tmp/openpilot/laika_repo -COPY ./rednose /tmp/openpilot/rednose -COPY ./tools /tmp/openpilot/tools -COPY ./release /tmp/openpilot/release -COPY ./common /tmp/openpilot/common -COPY ./opendbc /tmp/openpilot/opendbc -COPY ./cereal /tmp/openpilot/cereal -COPY ./panda /tmp/openpilot/panda -COPY ./selfdrive /tmp/openpilot/selfdrive + pipenv install --system --deploy --dev --clear && \ + pip uninstall -y pipenv diff --git a/Dockerfile.openpilotci b/Dockerfile.openpilotci new file mode 100644 index 0000000000..0490d0a2d0 --- /dev/null +++ b/Dockerfile.openpilotci @@ -0,0 +1,24 @@ +FROM commaai/openpilot-base:latest + +ENV PYTHONUNBUFFERED 1 +ENV PYTHONPATH /tmp/openpilot:${PYTHONPATH} + +RUN mkdir -p /tmp/openpilot + +COPY SConstruct \ + .pylintrc \ + .pre-commit-config.yaml \ + /tmp/openpilot/ + +COPY ./pyextra /tmp/openpilot/pyextra +COPY ./phonelibs /tmp/openpilot/phonelibs +COPY ./laika /tmp/openpilot/laika +COPY ./laika_repo /tmp/openpilot/laika_repo +COPY ./rednose /tmp/openpilot/rednose +COPY ./tools /tmp/openpilot/tools +COPY ./release /tmp/openpilot/release +COPY ./common /tmp/openpilot/common +COPY ./opendbc /tmp/openpilot/opendbc +COPY ./cereal /tmp/openpilot/cereal +COPY ./panda /tmp/openpilot/panda +COPY ./selfdrive /tmp/openpilot/selfdrive diff --git a/Jenkinsfile b/Jenkinsfile index dd0e2c12fa..eb16835ce5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -70,7 +70,7 @@ pipeline { stage('PC tests') { agent { dockerfile { - filename 'Dockerfile.openpilot' + filename 'Dockerfile.openpilotci' args '--privileged --shm-size=1G --user=root' } }