You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
431 lines
16 KiB
431 lines
16 KiB
name: selfdrive
|
|
on:
|
|
push:
|
|
branches-ignore:
|
|
- 'testing-closet*'
|
|
pull_request:
|
|
|
|
env:
|
|
BASE_IMAGE: openpilot-base
|
|
DOCKER_REGISTRY: ghcr.io/commaai
|
|
|
|
DOCKER_LOGIN: docker login ghcr.io -u adeebshihadeh -p ${{ secrets.CONTAINER_TOKEN }}
|
|
BUILD: |
|
|
docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) || true
|
|
docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
|
|
docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
|
|
|
|
RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache $BASE_IMAGE /bin/sh -c
|
|
UNIT_TEST: coverage run --append -m unittest discover
|
|
|
|
jobs:
|
|
# TODO: once actions/cache supports read only mode, use the cache for all jobs
|
|
build_release:
|
|
name: build release
|
|
runs-on: ubuntu-20.04
|
|
timeout-minutes: 50
|
|
env:
|
|
STRIPPED_DIR: tmppilot
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
with:
|
|
submodules: true
|
|
- name: Check submodules
|
|
if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
|
|
run: release/check-submodules.sh
|
|
- name: Cache scons
|
|
id: scons-cache
|
|
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
|
|
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
|
|
env:
|
|
CACHE_SKIP_SAVE: true
|
|
with:
|
|
path: /tmp/scons_cache
|
|
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
|
|
restore-keys: |
|
|
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
|
|
scons-
|
|
- name: Strip non-release files
|
|
run: |
|
|
mkdir $STRIPPED_DIR
|
|
cp -pR --parents $(cat release/files_common) $STRIPPED_DIR
|
|
cp -pR --parents $(cat release/files_tici) $STRIPPED_DIR
|
|
cp -pR --parents $(cat release/files_pc) $STRIPPED_DIR
|
|
cp Dockerfile.openpilot_base $STRIPPED_DIR
|
|
|
|
# need this to build on x86
|
|
cp -pR --parents third_party/libyuv third_party/snpe selfdrive/modeld/runners $STRIPPED_DIR
|
|
- name: Build Docker image
|
|
run: eval "$BUILD"
|
|
- name: Build openpilot and run checks
|
|
run: |
|
|
cd $STRIPPED_DIR
|
|
${{ env.RUN }} "CI=1 python selfdrive/manager/build.py && \
|
|
python -m unittest discover selfdrive/car"
|
|
- name: Cleanup scons cache
|
|
run: |
|
|
cd $STRIPPED_DIR
|
|
${{ env.RUN }} "scons -j$(nproc) && \
|
|
rm -rf /tmp/scons_cache/* && \
|
|
scons -j$(nproc) --cache-populate"
|
|
|
|
build_mac:
|
|
name: build macos
|
|
runs-on: macos-latest
|
|
timeout-minutes: 60
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
with:
|
|
submodules: true
|
|
- name: Determine pre-existing Homebrew packages
|
|
if: steps.dependency-cache.outputs.cache-hit != 'true'
|
|
run: |
|
|
echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
|
|
ls -1 /usr/local/Cellar >> $GITHUB_ENV
|
|
echo 'EOF' >> $GITHUB_ENV
|
|
- name: Cache dependencies
|
|
id: dependency-cache
|
|
uses: actions/cache@v2
|
|
with:
|
|
path: |
|
|
~/.pyenv
|
|
~/.local/share/virtualenvs/
|
|
/usr/local/Cellar
|
|
~/github_brew_cache_entries.txt
|
|
/tmp/scons_cache
|
|
key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'Pipfile*') }}
|
|
restore-keys: macos-
|
|
- name: Brew link restored dependencies
|
|
run: |
|
|
if [ -f ~/github_brew_cache_entries.txt ]; then
|
|
while read pkg; do
|
|
brew link --force "$pkg" # `--force` for keg-only packages
|
|
done < ~/github_brew_cache_entries.txt
|
|
else
|
|
echo "Cache entries not found"
|
|
fi
|
|
- name: Install dependencies
|
|
run: ./tools/mac_setup.sh
|
|
- name: Build openpilot
|
|
run: |
|
|
source tools/openpilot_env.sh
|
|
pipenv run selfdrive/manager/build.py
|
|
|
|
# cleanup scons cache
|
|
rm -rf /tmp/scons_cache/
|
|
pipenv run scons -j$(nproc) --cache-populate
|
|
- name: Remove pre-existing Homebrew packages for caching
|
|
if: steps.dependency-cache.outputs.cache-hit != 'true'
|
|
run: |
|
|
cd /usr/local/Cellar
|
|
new_cellar=$(ls -1)
|
|
comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
|
|
if [[ $pkg != "zstd" ]]; then # caching step needs zstd
|
|
rm -rf "$pkg"
|
|
fi
|
|
done
|
|
comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt
|
|
|
|
build_webcam:
|
|
name: build webcam
|
|
runs-on: ubuntu-20.04
|
|
timeout-minutes: 90
|
|
env:
|
|
IMAGE_NAME: openpilotwebcamci
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
with:
|
|
submodules: true
|
|
- name: Build Docker image
|
|
run: |
|
|
eval "$BUILD"
|
|
docker pull $DOCKER_REGISTRY/$IMAGE_NAME:latest || true
|
|
docker build --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/webcam/Dockerfile .
|
|
- name: Build openpilot
|
|
run: docker run --shm-size 1G --rm -v $PWD:/tmp/openpilot -e PYTHONPATH=/tmp/openpilot $DOCKER_REGISTRY/$IMAGE_NAME /bin/sh -c "cd /tmp/openpilot && USE_WEBCAM=1 scons -j$(nproc)"
|
|
- name: Push to container registry
|
|
if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
|
|
run: |
|
|
$DOCKER_LOGIN
|
|
docker push $DOCKER_REGISTRY/$IMAGE_NAME:latest
|
|
|
|
docker_push:
|
|
name: docker push
|
|
runs-on: ubuntu-20.04
|
|
timeout-minutes: 50
|
|
if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
|
|
needs: static_analysis # hack to ensure slow tests run first since this and static_analysis are fast
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
with:
|
|
submodules: true
|
|
- name: Build Docker image
|
|
run: eval "$BUILD"
|
|
- name: Push to container registry
|
|
run: |
|
|
$DOCKER_LOGIN
|
|
docker push $DOCKER_REGISTRY/$BASE_IMAGE:latest
|
|
|
|
static_analysis:
|
|
name: static analysis
|
|
runs-on: ubuntu-20.04
|
|
timeout-minutes: 50
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
with:
|
|
submodules: true
|
|
- name: Build Docker image
|
|
run: eval "$BUILD"
|
|
- name: pre-commit
|
|
run: ${{ env.RUN }} "git init && git add -A && pre-commit run --all"
|
|
|
|
valgrind:
|
|
name: valgrind
|
|
runs-on: ubuntu-20.04
|
|
timeout-minutes: 50
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
with:
|
|
submodules: true
|
|
- name: Cache dependencies
|
|
id: dependency-cache
|
|
uses: actions/cache@v2
|
|
with:
|
|
path: /tmp/comma_download_cache
|
|
key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/test_valgrind_replay.py') }}
|
|
- name: Cache scons
|
|
id: scons-cache
|
|
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
|
|
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
|
|
env:
|
|
CACHE_SKIP_SAVE: true
|
|
with:
|
|
path: /tmp/scons_cache
|
|
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
|
|
restore-keys: |
|
|
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
|
|
scons-
|
|
- name: Build Docker image
|
|
run: eval "$BUILD"
|
|
- name: Run valgrind
|
|
run: |
|
|
${{ env.RUN }} "scons -j$(nproc) && \
|
|
FILEREADER_CACHE=1 python selfdrive/test/test_valgrind_replay.py"
|
|
- name: Print logs
|
|
if: always()
|
|
run: cat selfdrive/test/valgrind_logs.txt
|
|
|
|
unit_tests:
|
|
name: unit tests
|
|
runs-on: ubuntu-20.04
|
|
timeout-minutes: 50
|
|
steps:
|
|
- name: Get current date
|
|
id: date
|
|
run: echo "::set-output name=time::$(date +'%s')"
|
|
- name: Output timestamp
|
|
run: echo $TIMESTAMP
|
|
env:
|
|
TIMESTAMP: ${{ steps.date.outputs.time }}
|
|
- uses: actions/checkout@v2
|
|
with:
|
|
submodules: true
|
|
- name: Cache scons
|
|
id: scons-cache
|
|
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
|
|
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
|
|
env:
|
|
CACHE_SKIP_SAVE: ${{ github.ref != 'refs/heads/master' || github.repository != 'commaai/openpilot' }}
|
|
with:
|
|
path: /tmp/scons_cache
|
|
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-${{ steps.date.outputs.time }}
|
|
restore-keys: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
|
|
- name: Build Docker image
|
|
run: eval "$BUILD"
|
|
- name: Run unit tests
|
|
run: |
|
|
${{ env.RUN }} "scons -j$(nproc) --test && \
|
|
coverage run selfdrive/test/test_fingerprints.py && \
|
|
$UNIT_TEST common && \
|
|
$UNIT_TEST opendbc/can && \
|
|
$UNIT_TEST selfdrive/boardd && \
|
|
$UNIT_TEST selfdrive/controls && \
|
|
$UNIT_TEST selfdrive/monitoring && \
|
|
$UNIT_TEST selfdrive/loggerd && \
|
|
$UNIT_TEST selfdrive/car && \
|
|
$UNIT_TEST selfdrive/locationd && \
|
|
$UNIT_TEST selfdrive/athena && \
|
|
$UNIT_TEST selfdrive/thermald && \
|
|
$UNIT_TEST selfdrive/hardware/tici && \
|
|
$UNIT_TEST tools/lib/tests && \
|
|
./selfdrive/boardd/tests/test_boardd_usbprotocol && \
|
|
./selfdrive/common/tests/test_util && \
|
|
./selfdrive/common/tests/test_swaglog && \
|
|
./selfdrive/loggerd/tests/test_logger &&\
|
|
./selfdrive/proclogd/tests/test_proclog && \
|
|
./selfdrive/ui/replay/tests/test_replay && \
|
|
./selfdrive/camerad/test/ae_gray_test && \
|
|
coverage xml"
|
|
- name: "Upload coverage to Codecov"
|
|
uses: codecov/codecov-action@v2
|
|
|
|
process_replay:
|
|
name: process replay
|
|
runs-on: ubuntu-20.04
|
|
timeout-minutes: 50
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
with:
|
|
submodules: true
|
|
- name: Cache dependencies
|
|
id: dependency-cache
|
|
uses: actions/cache@v2
|
|
with:
|
|
path: /tmp/comma_download_cache
|
|
key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/test_processes.py') }}
|
|
- name: Cache scons
|
|
id: scons-cache
|
|
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
|
|
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
|
|
env:
|
|
CACHE_SKIP_SAVE: true
|
|
with:
|
|
path: /tmp/scons_cache
|
|
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
|
|
restore-keys: |
|
|
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
|
|
scons-
|
|
- name: Build Docker image
|
|
run: eval "$BUILD"
|
|
- name: Run replay
|
|
run: |
|
|
${{ env.RUN }} "scons -j$(nproc) && \
|
|
FILEREADER_CACHE=1 CI=1 coverage run selfdrive/test/process_replay/test_processes.py && \
|
|
coverage xml"
|
|
- name: "Upload coverage to Codecov"
|
|
uses: codecov/codecov-action@v2
|
|
- name: Print diff
|
|
if: always()
|
|
run: cat selfdrive/test/process_replay/diff.txt
|
|
- uses: actions/upload-artifact@v2
|
|
if: always()
|
|
continue-on-error: true
|
|
with:
|
|
name: process_replay_diff.txt
|
|
path: selfdrive/test/process_replay/diff.txt
|
|
|
|
#model_replay:
|
|
# name: model replay
|
|
# runs-on: ubuntu-20.04
|
|
# timeout-minutes: 50
|
|
# steps:
|
|
# - uses: actions/checkout@v2
|
|
# with:
|
|
# submodules: true
|
|
# - name: Build Docker image
|
|
# run: eval "$BUILD"
|
|
# - name: Run replay
|
|
# run: |
|
|
# ${{ env.RUN }} "scons -j$(nproc) && \
|
|
# selfdrive/test/process_replay/model_replay.py"
|
|
|
|
test_longitudinal:
|
|
name: longitudinal
|
|
runs-on: ubuntu-20.04
|
|
timeout-minutes: 50
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
with:
|
|
submodules: true
|
|
- name: Cache scons
|
|
id: scons-cache
|
|
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
|
|
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
|
|
env:
|
|
CACHE_SKIP_SAVE: true
|
|
with:
|
|
path: /tmp/scons_cache
|
|
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
|
|
restore-keys: |
|
|
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
|
|
scons-
|
|
- name: Build Docker image
|
|
run: eval "$BUILD"
|
|
- name: Test longitudinal
|
|
run: |
|
|
${{ env.RUN }} "mkdir -p selfdrive/test/out && \
|
|
scons -j$(nproc) && \
|
|
cd selfdrive/test/longitudinal_maneuvers && \
|
|
coverage run ./test_longitudinal.py && \
|
|
coverage xml"
|
|
- name: "Upload coverage to Codecov"
|
|
uses: codecov/codecov-action@v2
|
|
- uses: actions/upload-artifact@v2
|
|
if: always()
|
|
continue-on-error: true
|
|
with:
|
|
name: longitudinal
|
|
path: selfdrive/test/longitudinal_maneuvers/out/longitudinal/
|
|
|
|
test_cars:
|
|
name: cars
|
|
runs-on: ubuntu-20.04
|
|
timeout-minutes: 50
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
job: [0, 1, 2, 3]
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
with:
|
|
submodules: true
|
|
- name: Cache dependencies
|
|
id: dependency-cache
|
|
uses: actions/cache@v2
|
|
with:
|
|
path: /tmp/comma_download_cache
|
|
key: car_models-${{ hashFiles('selfdrive/test/test_models.py', 'selfdrive/test/test_routes.py') }}-${{ matrix.job }}
|
|
- name: Cache scons
|
|
id: scons-cache
|
|
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
|
|
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
|
|
env:
|
|
CACHE_SKIP_SAVE: true
|
|
with:
|
|
path: /tmp/scons_cache
|
|
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
|
|
restore-keys: |
|
|
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
|
|
scons-
|
|
- name: Build Docker image
|
|
run: eval "$BUILD"
|
|
- name: Test car models
|
|
run: |
|
|
${{ env.RUN }} "scons -j$(nproc) --test && \
|
|
FILEREADER_CACHE=1 coverage run -m pytest selfdrive/test/test_models.py && \
|
|
coverage xml && \
|
|
chmod -R 777 /tmp/comma_download_cache"
|
|
env:
|
|
NUM_JOBS: 4
|
|
JOB_ID: ${{ matrix.job }}
|
|
- name: "Upload coverage to Codecov"
|
|
uses: codecov/codecov-action@v2
|
|
|
|
docs:
|
|
name: build docs
|
|
runs-on: ubuntu-20.04
|
|
timeout-minutes: 50
|
|
steps:
|
|
- uses: actions/checkout@v2
|
|
with:
|
|
submodules: true
|
|
- name: Build docker container
|
|
run: |
|
|
docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
|
|
docker pull $DOCKER_REGISTRY/openpilot-docs:latest || true
|
|
DOCKER_BUILDKIT=1 docker build --cache-from $DOCKER_REGISTRY/openpilot-docs:latest -t $DOCKER_REGISTRY/openpilot-docs:latest -f docs/docker/Dockerfile .
|
|
- name: Push docker container
|
|
if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
|
|
run: |
|
|
$DOCKER_LOGIN
|
|
docker push $DOCKER_REGISTRY/openpilot-docs:latest
|
|
|