openpilot is an open source driver assistance system. openpilot performs the functions of Automated Lane Centering and Adaptive Cruise Control for over 200 supported car makes and models.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

454 lines
17 KiB

name: selfdrive
on:
push:
branches-ignore:
- 'testing-closet*'
pull_request:
env:
BASE_IMAGE: openpilot-base
DOCKER_REGISTRY: ghcr.io/commaai
AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}
DOCKER_LOGIN: docker login ghcr.io -u adeebshihadeh -p ${{ secrets.CONTAINER_TOKEN }}
BUILD: |
docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) || true
docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache $BASE_IMAGE /bin/sh -c
UNIT_TEST: coverage run --append -m unittest discover
jobs:
# TODO: once actions/cache supports read only mode, use the cache for all jobs
build_release:
name: build release
runs-on: ubuntu-20.04
timeout-minutes: 50
env:
STRIPPED_DIR: /tmp/releasepilot
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true
- name: Check submodules
if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
run: release/check-submodules.sh
- name: Cache scons
id: scons-cache
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
env:
CACHE_SKIP_SAVE: true
with:
path: /tmp/scons_cache
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
restore-keys: |
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
scons-
- name: Build devel
run: |
TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
cp Dockerfile.openpilot_base $STRIPPED_DIR
- name: Build Docker image
run: eval "$BUILD"
- name: Build openpilot and run checks
run: |
cd $STRIPPED_DIR
${{ env.RUN }} "CI=1 python selfdrive/manager/build.py && \
python -m unittest discover selfdrive/car"
build_all:
name: build all
runs-on: ubuntu-20.04
timeout-minutes: 50
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Cache scons
id: scons-cache
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
env:
CACHE_SKIP_SAVE: ${{ github.ref != 'refs/heads/master' || github.repository != 'commaai/openpilot' }}
with:
path: /tmp/scons_cache
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-${{ steps.date.outputs.time }}
restore-keys: |
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
scons-
- name: Build Docker image
run: eval "$BUILD"
- name: Build openpilot with all flags
run: ${{ env.RUN }} "scons -j$(nproc) --extras --test"
- name: Cleanup scons cache
run: |
${{ env.RUN }} "scons -j$(nproc) --extras --test && \
rm -rf /tmp/scons_cache/* && \
scons -j$(nproc) --cache-populate"
#build_mac:
# name: build macos
# runs-on: macos-latest
# timeout-minutes: 60
# steps:
# - uses: actions/checkout@v3
# with:
# submodules: true
# - name: Determine pre-existing Homebrew packages
# if: steps.dependency-cache.outputs.cache-hit != 'true'
# run: |
# echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
# ls -1 /usr/local/Cellar >> $GITHUB_ENV
# echo 'EOF' >> $GITHUB_ENV
# - name: Cache dependencies
# id: dependency-cache
# uses: actions/cache@v2
# with:
# path: |
# ~/.pyenv
# ~/.local/share/virtualenvs/
# /usr/local/Cellar
# ~/github_brew_cache_entries.txt
# /tmp/scons_cache
# key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'Pipfile*') }}
# restore-keys: macos-
# - name: Brew link restored dependencies
# run: |
# if [ -f ~/github_brew_cache_entries.txt ]; then
# while read pkg; do
# brew link --force "$pkg" # `--force` for keg-only packages
# done < ~/github_brew_cache_entries.txt
# else
# echo "Cache entries not found"
# fi
# - name: Install dependencies
# run: ./tools/mac_setup.sh
# - name: Build openpilot
# run: |
# source tools/openpilot_env.sh
# pipenv run selfdrive/manager/build.py
#
# # cleanup scons cache
# rm -rf /tmp/scons_cache/
# pipenv run scons -j$(nproc) --cache-populate
# - name: Remove pre-existing Homebrew packages for caching
# if: steps.dependency-cache.outputs.cache-hit != 'true'
# run: |
# cd /usr/local/Cellar
# new_cellar=$(ls -1)
# comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
# if [[ $pkg != "zstd" ]]; then # caching step needs zstd
# rm -rf "$pkg"
# fi
# done
# comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt
build_webcam:
name: build webcam
runs-on: ubuntu-20.04
timeout-minutes: 90
env:
IMAGE_NAME: openpilotwebcamci
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Cache scons
id: scons-cache
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
env:
CACHE_SKIP_SAVE: true
with:
path: /tmp/scons_cache
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
restore-keys: |
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
scons-
- name: Build Docker image
run: |
eval "$BUILD"
docker pull $DOCKER_REGISTRY/$IMAGE_NAME:latest || true
docker build --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/webcam/Dockerfile .
- name: Build openpilot
run: docker run --shm-size 1G --rm -v $PWD:/tmp/openpilot -e PYTHONPATH=/tmp/openpilot $DOCKER_REGISTRY/$IMAGE_NAME /bin/sh -c "cd /tmp/openpilot && USE_WEBCAM=1 scons -j$(nproc)"
- name: Push to container registry
if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
run: |
$DOCKER_LOGIN
docker push $DOCKER_REGISTRY/$IMAGE_NAME:latest
docker_push:
name: docker push
runs-on: ubuntu-20.04
timeout-minutes: 50
if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
needs: static_analysis # hack to ensure slow tests run first since this and static_analysis are fast
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Build Docker image
run: eval "$BUILD"
- name: Push to container registry
run: |
$DOCKER_LOGIN
docker push $DOCKER_REGISTRY/$BASE_IMAGE:latest
static_analysis:
name: static analysis
runs-on: ubuntu-20.04
timeout-minutes: 50
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Build Docker image
run: eval "$BUILD"
- name: pre-commit
run: ${{ env.RUN }} "pre-commit run --all"
valgrind:
name: valgrind
runs-on: ubuntu-20.04
timeout-minutes: 50
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Cache dependencies
id: dependency-cache
uses: actions/cache@v2
with:
path: /tmp/comma_download_cache
key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/test_valgrind_replay.py') }}
- name: Cache scons
id: scons-cache
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
env:
CACHE_SKIP_SAVE: true
with:
path: /tmp/scons_cache
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
restore-keys: |
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
scons-
- name: Build Docker image
run: eval "$BUILD"
- name: Run valgrind
run: |
${{ env.RUN }} "scons -j$(nproc) && \
FILEREADER_CACHE=1 python selfdrive/test/test_valgrind_replay.py"
- name: Print logs
if: always()
run: cat selfdrive/test/valgrind_logs.txt
unit_tests:
name: unit tests
runs-on: ubuntu-20.04
timeout-minutes: 50
steps:
- name: Get current date
id: date
run: echo "::set-output name=time::$(date +'%s')"
- name: Output timestamp
run: echo $TIMESTAMP
env:
TIMESTAMP: ${{ steps.date.outputs.time }}
- uses: actions/checkout@v3
with:
submodules: true
- name: Cache scons
id: scons-cache
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
env:
CACHE_SKIP_SAVE: ${{ github.ref != 'refs/heads/master' || github.repository != 'commaai/openpilot' }}
with:
path: /tmp/scons_cache
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-${{ steps.date.outputs.time }}
restore-keys: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
- name: Build Docker image
run: eval "$BUILD"
- name: Run unit tests
run: |
${{ env.RUN }} "export SKIP_LONG_TESTS=1 && \
scons -j$(nproc) --test && \
$UNIT_TEST common && \
$UNIT_TEST opendbc/can && \
$UNIT_TEST selfdrive/boardd && \
$UNIT_TEST selfdrive/controls && \
$UNIT_TEST selfdrive/monitoring && \
$UNIT_TEST selfdrive/loggerd && \
$UNIT_TEST selfdrive/car && \
$UNIT_TEST selfdrive/locationd && \
$UNIT_TEST selfdrive/athena && \
$UNIT_TEST selfdrive/thermald && \
$UNIT_TEST selfdrive/hardware/tici && \
$UNIT_TEST selfdrive/modeld && \
$UNIT_TEST tools/lib/tests && \
./common/tests/test_util && \
./common/tests/test_swaglog && \
./selfdrive/boardd/tests/test_boardd_usbprotocol && \
./selfdrive/loggerd/tests/test_logger &&\
./system/proclogd/tests/test_proclog && \
./selfdrive/ui/replay/tests/test_replay && \
./selfdrive/camerad/test/ae_gray_test && \
coverage xml"
- name: "Upload coverage to Codecov"
uses: codecov/codecov-action@v2
process_replay:
name: process replay
runs-on: ubuntu-20.04
timeout-minutes: 50
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Cache dependencies
id: dependency-cache
uses: actions/cache@v2
with:
path: /tmp/comma_download_cache
key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/test_processes.py') }}
- name: Cache scons
id: scons-cache
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
env:
CACHE_SKIP_SAVE: true
with:
path: /tmp/scons_cache
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
restore-keys: |
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
scons-
- name: Build Docker image
run: eval "$BUILD"
- name: Run replay
run: |
${{ env.RUN }} "scons -j$(nproc) && \
FILEREADER_CACHE=1 CI=1 coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
coverage xml"
- name: Print diff
if: always()
run: cat selfdrive/test/process_replay/diff.txt
- uses: actions/upload-artifact@v2
if: always()
continue-on-error: true
with:
name: process_replay_diff.txt
path: selfdrive/test/process_replay/diff.txt
- name: Upload reference logs
if: ${{ failure() && github.event_name == 'pull_request' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
run: |
${{ env.RUN }} "scons -j$(nproc) && \
CI=1 AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
- name: "Upload coverage to Codecov"
uses: codecov/codecov-action@v2
test_longitudinal:
name: longitudinal
runs-on: ubuntu-20.04
timeout-minutes: 50
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Cache scons
id: scons-cache
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
env:
CACHE_SKIP_SAVE: true
with:
path: /tmp/scons_cache
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
restore-keys: |
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
scons-
- name: Build Docker image
run: eval "$BUILD"
- name: Test longitudinal
run: |
${{ env.RUN }} "mkdir -p selfdrive/test/out && \
scons -j$(nproc) && \
cd selfdrive/test/longitudinal_maneuvers && \
coverage run ./test_longitudinal.py && \
coverage xml"
- name: "Upload coverage to Codecov"
uses: codecov/codecov-action@v2
- uses: actions/upload-artifact@v2
if: always()
continue-on-error: true
with:
name: longitudinal
path: selfdrive/test/longitudinal_maneuvers/out/longitudinal/
test_cars:
name: cars
runs-on: ubuntu-20.04
timeout-minutes: 50
strategy:
fail-fast: false
matrix:
job: [0, 1, 2, 3, 4]
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Cache dependencies
id: dependency-cache
uses: actions/cache@v2
with:
path: /tmp/comma_download_cache
key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }}
- name: Cache scons
id: scons-cache
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
env:
CACHE_SKIP_SAVE: true
with:
path: /tmp/scons_cache
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
restore-keys: |
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
scons-
- name: Build Docker image
run: eval "$BUILD"
- name: Test car models
run: |
${{ env.RUN }} "scons -j$(nproc) --test && \
FILEREADER_CACHE=1 coverage run -m pytest selfdrive/car/tests/test_models.py && \
coverage xml && \
chmod -R 777 /tmp/comma_download_cache"
env:
NUM_JOBS: 5
JOB_ID: ${{ matrix.job }}
- name: "Upload coverage to Codecov"
uses: codecov/codecov-action@v2
docs:
name: build docs
runs-on: ubuntu-20.04
timeout-minutes: 50
steps:
- uses: actions/checkout@v3
with:
submodules: true
- name: Build docker container
run: |
docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
docker pull $DOCKER_REGISTRY/openpilot-docs:latest || true
DOCKER_BUILDKIT=1 docker build --cache-from $DOCKER_REGISTRY/openpilot-docs:latest -t $DOCKER_REGISTRY/openpilot-docs:latest -f docs/docker/Dockerfile .
- name: Push docker container
if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
run: |
$DOCKER_LOGIN
docker push $DOCKER_REGISTRY/openpilot-docs:latest