openpilot is an open source driver assistance system. openpilot performs the functions of Automated Lane Centering and Adaptive Cruise Control for over 200 supported car makes and models.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

585 lines
23 KiB

name: selfdrive
on:
push:
branches-ignore:
- 'testing-closet*'
pull_request:
env:
BASE_IMAGE: openpilot-base
CL_BASE_IMAGE: openpilot-base-cl
DOCKER_REGISTRY: ghcr.io/commaai
AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}
DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
BUILD: |
docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) || true
docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
BUILD_CL: |
docker pull $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest || true
docker build --cache-from $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $CL_BASE_IMAGE:latest -f Dockerfile.openpilot_base_cl .
RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
UNIT_TEST: coverage run --append -m unittest discover
jobs:
# TODO: once actions/cache supports read only mode, use the cache for all jobs
build_release:
name: build release
runs-on: ubuntu-20.04
timeout-minutes: 50
env:
STRIPPED_DIR: /tmp/releasepilot
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true
- name: Pull LFS
run: git lfs pull
- name: Check submodules
if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
run: release/check-submodules.sh
- name: Cache scons
id: scons-cache
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
env:
CACHE_SKIP_SAVE: true
with:
path: /tmp/scons_cache
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
restore-keys: |
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
scons-
- name: Build devel
run: |
TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
cp Dockerfile.openpilot_base $STRIPPED_DIR
cp .pre-commit-config.yaml $STRIPPED_DIR
cp .pylintrc $STRIPPED_DIR
cp mypy.ini $STRIPPED_DIR
- name: Build Docker image
run: |
eval "$BUILD"
rm $STRIPPED_DIR/Dockerfile.openpilot_base
- name: Build openpilot and run checks
run: |
cd $STRIPPED_DIR
${{ env.RUN }} "CI=1 python selfdrive/manager/build.py && \
pre-commit run --all && \
rm .pre-commit-config.yaml && \
rm .pylintrc && \
rm mypy.ini && \
release/check-dirty.sh && \
python -m unittest discover selfdrive/car"
build_all:
name: build all
runs-on: ubuntu-20.04
timeout-minutes: 50
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Cache scons
id: scons-cache
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
env:
CACHE_SKIP_SAVE: ${{ github.ref != 'refs/heads/master' || github.repository != 'commaai/openpilot' }}
with:
path: /tmp/scons_cache
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-${{ steps.date.outputs.time }}
restore-keys: |
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
scons-
- name: Build Docker image
run: eval "$BUILD"
- name: Build openpilot with all flags
run: ${{ env.RUN }} "scons -j$(nproc) --extras --test && release/check-dirty.sh"
- name: Cleanup scons cache
run: |
${{ env.RUN }} "scons -j$(nproc) --extras --test && \
rm -rf /tmp/scons_cache/* && \
scons -j$(nproc) --extras --test --cache-populate"
#build_mac:
# name: build macos
# runs-on: macos-latest
# timeout-minutes: 60
# steps:
# - uses: actions/checkout@v3
# with:
# submodules: true
# - name: Determine pre-existing Homebrew packages
# if: steps.dependency-cache.outputs.cache-hit != 'true'
# run: |
# echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
# ls -1 /usr/local/Cellar >> $GITHUB_ENV
# echo 'EOF' >> $GITHUB_ENV
# - name: Cache dependencies
# id: dependency-cache
# uses: actions/cache@v2
# with:
# path: |
# ~/.pyenv
# ~/.local/share/virtualenvs/
# /usr/local/Cellar
# ~/github_brew_cache_entries.txt
# /tmp/scons_cache
# key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'Pipfile*') }}
# restore-keys: macos-
# - name: Brew link restored dependencies
# run: |
# if [ -f ~/github_brew_cache_entries.txt ]; then
# while read pkg; do
# brew link --force "$pkg" # `--force` for keg-only packages
# done < ~/github_brew_cache_entries.txt
# else
# echo "Cache entries not found"
# fi
# - name: Install dependencies
# run: ./tools/mac_setup.sh
# - name: Build openpilot
# run: |
# source tools/openpilot_env.sh
# pipenv run selfdrive/manager/build.py
#
# # cleanup scons cache
# rm -rf /tmp/scons_cache/
# pipenv run scons -j$(nproc) --cache-populate
# - name: Remove pre-existing Homebrew packages for caching
# if: steps.dependency-cache.outputs.cache-hit != 'true'
# run: |
# cd /usr/local/Cellar
# new_cellar=$(ls -1)
# comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
# if [[ $pkg != "zstd" ]]; then # caching step needs zstd
# rm -rf "$pkg"
# fi
# done
# comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt
build_webcam:
name: build webcam
runs-on: ubuntu-20.04
timeout-minutes: 90
env:
IMAGE_NAME: openpilotwebcamci
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Cache scons
id: scons-cache
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
env:
CACHE_SKIP_SAVE: true
with:
path: /tmp/scons_cache
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
restore-keys: |
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
scons-
- name: Build Docker image
run: |
eval "$BUILD"
docker pull $DOCKER_REGISTRY/$IMAGE_NAME:latest || true
docker build --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/webcam/Dockerfile .
- name: Build openpilot
run: docker run --shm-size 1G --rm -v $PWD:/tmp/openpilot -e PYTHONPATH=/tmp/openpilot $DOCKER_REGISTRY/$IMAGE_NAME /bin/sh -c "cd /tmp/openpilot && USE_WEBCAM=1 scons -j$(nproc)"
- name: Push to container registry
if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
run: |
$DOCKER_LOGIN
docker push $DOCKER_REGISTRY/$IMAGE_NAME:latest
docker_push:
name: docker push
runs-on: ubuntu-20.04
timeout-minutes: 50
if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
needs: static_analysis # hack to ensure slow tests run first since this and static_analysis are fast
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Build Docker image
run: eval "$BUILD"
- name: Push to container registry
run: |
$DOCKER_LOGIN
docker push $DOCKER_REGISTRY/$BASE_IMAGE:latest
- name: Build CL Docker image
run: eval "$BUILD_CL"
- name: Push to container registry
run: |
$DOCKER_LOGIN
docker push $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest
static_analysis:
name: static analysis
runs-on: ubuntu-20.04
timeout-minutes: 50
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Build Docker image
run: eval "$BUILD"
- name: pre-commit
run: ${{ env.RUN }} "pre-commit run --all"
valgrind:
name: valgrind
runs-on: ubuntu-20.04
timeout-minutes: 50
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Cache dependencies
id: dependency-cache
uses: actions/cache@v2
with:
path: /tmp/comma_download_cache
key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/test_valgrind_replay.py') }}
- name: Cache scons
id: scons-cache
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
env:
CACHE_SKIP_SAVE: true
with:
path: /tmp/scons_cache
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
restore-keys: |
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
scons-
- name: Build Docker image
run: eval "$BUILD"
- name: Run valgrind
run: |
${{ env.RUN }} "scons -j$(nproc) && \
FILEREADER_CACHE=1 python selfdrive/test/test_valgrind_replay.py"
- name: Print logs
if: always()
run: cat selfdrive/test/valgrind_logs.txt
unit_tests:
name: unit tests
runs-on: ubuntu-20.04
timeout-minutes: 50
steps:
- name: Get current date
id: date
run: echo "::set-output name=time::$(date +'%s')"
- name: Output timestamp
run: echo $TIMESTAMP
env:
TIMESTAMP: ${{ steps.date.outputs.time }}
- uses: actions/checkout@v3
with:
submodules: true
- name: Cache scons
id: scons-cache
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
env:
CACHE_SKIP_SAVE: ${{ github.ref != 'refs/heads/master' || github.repository != 'commaai/openpilot' }}
with:
path: /tmp/scons_cache
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-${{ steps.date.outputs.time }}
restore-keys: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
- name: Build Docker image
run: eval "$BUILD"
- name: Run unit tests
run: |
${{ env.RUN }} "export SKIP_LONG_TESTS=1 && \
scons -j$(nproc) --test && \
$UNIT_TEST common && \
$UNIT_TEST opendbc/can && \
$UNIT_TEST selfdrive/boardd && \
$UNIT_TEST selfdrive/controls && \
$UNIT_TEST selfdrive/monitoring && \
$UNIT_TEST selfdrive/loggerd && \
$UNIT_TEST selfdrive/car && \
$UNIT_TEST selfdrive/locationd && \
selfdrive/locationd/test/_test_locationd_lib.py && \
$UNIT_TEST selfdrive/athena && \
$UNIT_TEST selfdrive/thermald && \
$UNIT_TEST system/hardware/tici && \
$UNIT_TEST selfdrive/modeld && \
$UNIT_TEST tools/lib/tests && \
./selfdrive/ui/tests/create_test_translations.sh && \
QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
./selfdrive/ui/tests/test_translations.py && \
./common/tests/test_util && \
./common/tests/test_swaglog && \
./selfdrive/boardd/tests/test_boardd_usbprotocol && \
./selfdrive/loggerd/tests/test_logger &&\
./system/proclogd/tests/test_proclog && \
./tools/replay/tests/test_replay && \
./system/camerad/test/ae_gray_test && \
coverage xml"
- name: "Upload coverage to Codecov"
uses: codecov/codecov-action@v2
process_replay:
name: process replay
runs-on: ubuntu-20.04
timeout-minutes: 50
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Cache dependencies
id: dependency-cache
uses: actions/cache@v2
with:
path: /tmp/comma_download_cache
key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/test_processes.py') }}
- name: Cache scons
id: scons-cache
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
env:
CACHE_SKIP_SAVE: true
with:
path: /tmp/scons_cache
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
restore-keys: |
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
scons-
- name: Build Docker image
run: eval "$BUILD"
- name: Run replay
run: |
${{ env.RUN }} "scons -j$(nproc) && \
FILEREADER_CACHE=1 CI=1 coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
coverage xml"
- name: Print diff
if: always()
run: cat selfdrive/test/process_replay/diff.txt
- uses: actions/upload-artifact@v2
if: always()
continue-on-error: true
with:
name: process_replay_diff.txt
path: selfdrive/test/process_replay/diff.txt
- name: Upload reference logs
if: ${{ failure() && github.event_name == 'pull_request' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
run: |
${{ env.RUN }} "scons -j$(nproc) && \
CI=1 AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
- name: "Upload coverage to Codecov"
uses: codecov/codecov-action@v2
model_replay_onnx:
name: model replay onnx
runs-on: ubuntu-20.04
timeout-minutes: 50
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Pull LFS
run: git lfs pull
- name: Cache dependencies
id: dependency-cache
uses: actions/cache@v2
with:
path: /tmp/comma_download_cache
key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/model_replay.py') }}
- name: Cache scons
id: scons-cache
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
env:
CACHE_SKIP_SAVE: true
with:
path: /tmp/scons_cache
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
restore-keys: |
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
scons-
- name: Build Docker image
# Sim docker is needed to get the intel OPENCL drivers
run: eval "$BUILD_CL"
- name: Run replay
run: |
${{ env.RUN_CL }} "scons -j$(nproc) && \
ONNXCPU=1 FILEREADER_CACHE=1 CI=1 coverage run \
selfdrive/test/process_replay/model_replay.py -j$(nproc) && \
coverage xml"
test_longitudinal:
name: longitudinal
runs-on: ubuntu-20.04
timeout-minutes: 50
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Cache scons
id: scons-cache
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
env:
CACHE_SKIP_SAVE: true
with:
path: /tmp/scons_cache
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
restore-keys: |
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
scons-
- name: Build Docker image
run: eval "$BUILD"
- name: Test longitudinal
run: |
${{ env.RUN }} "mkdir -p selfdrive/test/out && \
scons -j$(nproc) && \
cd selfdrive/test/longitudinal_maneuvers && \
coverage run ./test_longitudinal.py && \
coverage xml"
- name: "Upload coverage to Codecov"
uses: codecov/codecov-action@v2
- uses: actions/upload-artifact@v2
if: always()
continue-on-error: true
with:
name: longitudinal
path: selfdrive/test/longitudinal_maneuvers/out/longitudinal/
test_cars:
name: cars
runs-on: ubuntu-20.04
timeout-minutes: 50
strategy:
fail-fast: false
matrix:
job: [0, 1, 2, 3, 4]
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Cache dependencies
id: dependency-cache
uses: actions/cache@v2
with:
path: /tmp/comma_download_cache
key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }}
- name: Cache scons
id: scons-cache
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
env:
CACHE_SKIP_SAVE: true
with:
path: /tmp/scons_cache
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
restore-keys: |
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
scons-
- name: Build Docker image
run: eval "$BUILD"
- name: Test car models
run: |
${{ env.RUN }} "scons -j$(nproc) --test && \
FILEREADER_CACHE=1 coverage run -m pytest selfdrive/car/tests/test_models.py && \
coverage xml && \
chmod -R 777 /tmp/comma_download_cache"
env:
NUM_JOBS: 5
JOB_ID: ${{ matrix.job }}
- name: "Upload coverage to Codecov"
uses: codecov/codecov-action@v2
docs:
name: build docs
runs-on: ubuntu-20.04
timeout-minutes: 50
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Build docker container
run: |
docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
docker pull $DOCKER_REGISTRY/openpilot-docs:latest || true
DOCKER_BUILDKIT=1 docker build --cache-from $DOCKER_REGISTRY/openpilot-docs:latest -t $DOCKER_REGISTRY/openpilot-docs:latest -f docs/docker/Dockerfile .
- name: Push docker container
if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
run: |
$DOCKER_LOGIN
docker push $DOCKER_REGISTRY/openpilot-docs:latest
car_docs_diff:
name: comment on PR with car docs diff
runs-on: ubuntu-20.04
timeout-minutes: 50
if: github.event_name == 'pull_request'
steps:
- uses: actions/checkout@v3
with:
submodules: true
ref: ${{ github.event.pull_request.base.ref }}
- name: Cache scons
id: scons-cache
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
env:
CACHE_SKIP_SAVE: true
with:
path: /tmp/scons_cache
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
restore-keys: |
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
scons-
- name: Build Docker image
run: eval "$BUILD"
- name: Get base car info
run: |
${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_info.py --path /tmp/openpilot_cache/base_car_info"
sudo chown -R $USER:$USER ${{ github.workspace }}
- uses: actions/checkout@v3
with:
submodules: true
- name: Save car docs diff
id: save_diff
run: |
${{ env.RUN }} "scons -j$(nproc)"
output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_info") || true
output="${output//$'\n'/'%0A'}"
echo "::set-output name=diff::$output"
- name: Find comment
if: ${{ env.AZURE_TOKEN != '' }}
uses: peter-evans/find-comment@1769778a0c5bd330272d749d12c036d65e70d39d
id: fc
with:
issue-number: ${{ github.event.pull_request.number }}
body-includes: This PR makes changes to
- name: Update comment
if: ${{ steps.save_diff.outputs.diff != '' && env.AZURE_TOKEN != '' }}
uses: peter-evans/create-or-update-comment@b95e16d2859ad843a14218d1028da5b2c4cbc4b4
with:
comment-id: ${{ steps.fc.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: "${{ steps.save_diff.outputs.diff }}"
edit-mode: replace
- name: Delete comment
if: ${{ steps.fc.outputs.comment-id != '' && steps.save_diff.outputs.diff == '' && env.AZURE_TOKEN != '' }}
uses: actions/github-script@v6
with:
script: |
github.rest.issues.deleteComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: ${{ steps.fc.outputs.comment-id }}
})