sadmen
/
openpilot_comma
mirror of https://github.com/commaai/openpilot.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CI tuneup (#24541)

Browse Source 
* little housekeeping

* all the commits

* one more cars
pull/24542/head
Adeeb Shihadeh 3 years ago committed by GitHub
parent 6a48a7acef
commit b2569d871f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 64 additions and 88 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      .github/workflows/prebuilt.yaml
  2. 93
      .github/workflows/selfdrive_tests.yaml
  3. 30
      .github/workflows/tools_tests.yaml
  4. 21
      Jenkinsfile
  5. 6
      release/files_pc

2
.github/workflows/prebuilt.yaml
Unescape View File

@ -30,7 +30,7 @@ jobs:
ref: master ref: master
wait-interval: 30 wait-interval: 30
running-workflow-name: 'build prebuilt' running-workflow-name: 'build prebuilt'
- uses: actions/checkout@v2 - uses: actions/checkout@v3
with: with:
submodules: true submodules: true
- name: Build Docker image - name: Build Docker image

93
.github/workflows/selfdrive_tests.yaml
Unescape View File

@ -26,10 +26,11 @@ jobs:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
timeout-minutes: 50 timeout-minutes: 50
env: env:
STRIPPED_DIR: tmppilot STRIPPED_DIR: /tmp/releasepilot
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
with: with:
fetch-depth: 0
submodules: true submodules: true
- name: Check submodules - name: Check submodules
if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot' if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
@ -46,16 +47,10 @@ jobs:
restore-keys: | restore-keys: |
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
scons- scons-
- name: Strip non-release files - name: Build devel
run: | run: |
mkdir $STRIPPED_DIR TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
cp -pR --parents $(cat release/files_common) $STRIPPED_DIR
cp -pR --parents $(cat release/files_tici) $STRIPPED_DIR
cp -pR --parents $(cat release/files_pc) $STRIPPED_DIR
cp Dockerfile.openpilot_base $STRIPPED_DIR cp Dockerfile.openpilot_base $STRIPPED_DIR
# need this to build on x86
cp -pR --parents third_party/libyuv third_party/snpe selfdrive/modeld/runners $STRIPPED_DIR
- name: Build Docker image - name: Build Docker image
run: eval "$BUILD" run: eval "$BUILD"
- name: Build openpilot and run checks - name: Build openpilot and run checks
@ -63,32 +58,43 @@ jobs:
cd $STRIPPED_DIR cd $STRIPPED_DIR
${{ env.RUN }} "CI=1 python selfdrive/manager/build.py && \ ${{ env.RUN }} "CI=1 python selfdrive/manager/build.py && \
python -m unittest discover selfdrive/car" python -m unittest discover selfdrive/car"
- name: Cleanup scons cache
run: |
cd $STRIPPED_DIR
${{ env.RUN }} "scons -j$(nproc) && \
rm -rf /tmp/scons_cache/* && \
scons -j$(nproc) --cache-populate"
build_all: build_all:
name: build all name: build all
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
timeout-minutes: 50 timeout-minutes: 50
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
with: with:
submodules: true submodules: true
- name: Cache scons
id: scons-cache
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
env:
CACHE_SKIP_SAVE: ${{ github.ref != 'refs/heads/master' || github.repository != 'commaai/openpilot' }}
with:
path: /tmp/scons_cache
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-${{ steps.date.outputs.time }}
restore-keys: |
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
scons-
- name: Build Docker image - name: Build Docker image
run: eval "$BUILD" run: eval "$BUILD"
- name: Build openpilot with all flags - name: Build openpilot with all flags
run: ${{ env.RUN }} "scons -j$(nproc) --extras --test" run: ${{ env.RUN }} "scons -j$(nproc) --extras --test"
- name: Cleanup scons cache
run: |
${{ env.RUN }} "scons -j$(nproc) --extras --test && \
rm -rf /tmp/scons_cache/* && \
scons -j$(nproc) --cache-populate"
#build_mac: #build_mac:
# name: build macos # name: build macos
# runs-on: macos-latest # runs-on: macos-latest
# timeout-minutes: 60 # timeout-minutes: 60
# steps: # steps:
# - uses: actions/checkout@v2 # - uses: actions/checkout@v3
# with: # with:
# submodules: true # submodules: true
# - name: Determine pre-existing Homebrew packages # - name: Determine pre-existing Homebrew packages
@ -147,9 +153,21 @@ jobs:
env: env:
IMAGE_NAME: openpilotwebcamci IMAGE_NAME: openpilotwebcamci
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
with: with:
submodules: true submodules: true
- name: Cache scons
id: scons-cache
# TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
env:
CACHE_SKIP_SAVE: true
with:
path: /tmp/scons_cache
key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
restore-keys: |
scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
scons-
- name: Build Docker image - name: Build Docker image
run: | run: |
eval "$BUILD" eval "$BUILD"
@ -170,7 +188,7 @@ jobs:
if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
needs: static_analysis # hack to ensure slow tests run first since this and static_analysis are fast needs: static_analysis # hack to ensure slow tests run first since this and static_analysis are fast
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
with: with:
submodules: true submodules: true
- name: Build Docker image - name: Build Docker image
@ -185,20 +203,20 @@ jobs:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
timeout-minutes: 50 timeout-minutes: 50
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
with: with:
submodules: true submodules: true
- name: Build Docker image - name: Build Docker image
run: eval "$BUILD" run: eval "$BUILD"
- name: pre-commit - name: pre-commit
run: ${{ env.RUN }} "git init && git add -A && pre-commit run --all" run: ${{ env.RUN }} "pre-commit run --all"
valgrind: valgrind:
name: valgrind name: valgrind
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
timeout-minutes: 50 timeout-minutes: 50
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
with: with:
submodules: true submodules: true
- name: Cache dependencies - name: Cache dependencies
@ -241,7 +259,7 @@ jobs:
run: echo $TIMESTAMP run: echo $TIMESTAMP
env: env:
TIMESTAMP: ${{ steps.date.outputs.time }} TIMESTAMP: ${{ steps.date.outputs.time }}
- uses: actions/checkout@v2 - uses: actions/checkout@v3
with: with:
submodules: true submodules: true
- name: Cache scons - name: Cache scons
@ -289,7 +307,7 @@ jobs:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
timeout-minutes: 50 timeout-minutes: 50
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
with: with:
submodules: true submodules: true
- name: Cache dependencies - name: Cache dependencies
@ -334,27 +352,12 @@ jobs:
- name: "Upload coverage to Codecov" - name: "Upload coverage to Codecov"
uses: codecov/codecov-action@v2 uses: codecov/codecov-action@v2
#model_replay:
# name: model replay
# runs-on: ubuntu-20.04
# timeout-minutes: 50
# steps:
# - uses: actions/checkout@v2
# with:
# submodules: true
# - name: Build Docker image
# run: eval "$BUILD"
# - name: Run replay
# run: |
# ${{ env.RUN }} "scons -j$(nproc) && \
# selfdrive/test/process_replay/model_replay.py"
test_longitudinal: test_longitudinal:
name: longitudinal name: longitudinal
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
timeout-minutes: 50 timeout-minutes: 50
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
with: with:
submodules: true submodules: true
- name: Cache scons - name: Cache scons
@ -394,9 +397,9 @@ jobs:
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
job: [0, 1, 2, 3] job: [0, 1, 2, 3, 4]
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
with: with:
submodules: true submodules: true
- name: Cache dependencies - name: Cache dependencies
@ -426,7 +429,7 @@ jobs:
coverage xml && \ coverage xml && \
chmod -R 777 /tmp/comma_download_cache" chmod -R 777 /tmp/comma_download_cache"
env: env:
NUM_JOBS: 4 NUM_JOBS: 5
JOB_ID: ${{ matrix.job }} JOB_ID: ${{ matrix.job }}
- name: "Upload coverage to Codecov" - name: "Upload coverage to Codecov"
uses: codecov/codecov-action@v2 uses: codecov/codecov-action@v2
@ -436,7 +439,7 @@ jobs:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
timeout-minutes: 50 timeout-minutes: 50
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
with: with:
submodules: true submodules: true
- name: Build docker container - name: Build docker container

30
.github/workflows/tools_tests.yaml
Unescape View File

@ -21,7 +21,7 @@ jobs:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
timeout-minutes: 30 timeout-minutes: 30
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
with: with:
submodules: true submodules: true
- name: Build Docker image - name: Build Docker image
@ -42,28 +42,16 @@ jobs:
IMAGE_NAME: openpilot-sim IMAGE_NAME: openpilot-sim
if: github.repository == 'commaai/openpilot' if: github.repository == 'commaai/openpilot'
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v3
with: with:
lfs: true
submodules: true submodules: true
- name: Build base image
# HACK: cache LFS objects since they count against our quota run: eval "$BUILD"
# https://github.com/actions/checkout/issues/165#issuecomment-657673315 - name: Pull latest simulator image
- name: Create LFS file list run: eval "$BUILD"docker pull $DOCKER_REGISTRY/$IMAGE_NAME:latest || true
run: git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id - name: Build simulator image
- name: Restore LFS cache run: docker build --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/sim/Dockerfile.sim .
uses: actions/cache@v2
id: lfs-cache
with:
path: .git/lfs
key: ${{ runner.os }}-lfs-${{ hashFiles('.lfs-assets-id') }}
- name: Git LFS Pull
run: git lfs pull
- name: Build Docker image
run: |
eval "$BUILD"
docker pull $DOCKER_REGISTRY/$IMAGE_NAME:latest || true
docker build --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/sim/Dockerfile.sim .
- name: Push to container registry - name: Push to container registry
if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot' if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
run: | run: |

21
Jenkinsfile vendored
Unescape View File

@ -80,27 +80,6 @@ pipeline {
stages { stages {
stage('parallel tests') { stage('parallel tests') {
parallel { parallel {
/*
stage('Power Consumption Tests') {
steps {
lock(resource: "", label: "c2-zookeeper", inversePrecedence: true, variable: 'device_ip', quantity: 1) {
timeout(time: 90, unit: 'MINUTES') {
sh script: "/home/batman/tools/zookeeper/enable_and_wait.py $device_ip 120", label: "turn on device"
phone(device_ip, "git checkout", readFile("selfdrive/test/setup_device_ci.sh"),)
phone(device_ip, "build", "scons -j4 && sync")
sh script: "/home/batman/tools/zookeeper/disable.py $device_ip", label: "turn off device"
sh script: "/home/batman/tools/zookeeper/enable_and_wait.py $device_ip 120", label: "turn on device"
sh script: "/home/batman/tools/zookeeper/check_consumption.py 60 3", label: "idle power consumption after boot"
sh script: "/home/batman/tools/zookeeper/ignition.py 1", label: "go onroad"
sh script: "/home/batman/tools/zookeeper/check_consumption.py 60 10", label: "onroad power consumption"
sh script: "/home/batman/tools/zookeeper/ignition.py 0", label: "go offroad"
sh script: "/home/batman/tools/zookeeper/check_consumption.py 60 2", label: "idle power consumption offroad"
}
}
}
}
*/
stage('build') { stage('build') {
environment { environment {
R3_PUSH = "${env.BRANCH_NAME == 'master' ? '1' : ' '}" R3_PUSH = "${env.BRANCH_NAME == 'master' ? '1' : ' '}"

6
release/files_pc
Unescape View File

@ -1,3 +1,9 @@
selfdrive/modeld/runners/onnx*
third_party/mapbox-gl-native-qt/x86_64/*.so third_party/mapbox-gl-native-qt/x86_64/*.so
third_party/qt-plugins/x86_64/geoservices/*.so third_party/qt-plugins/x86_64/geoservices/*.so
third_party/libyuv/x64/**
third_party/snpe/x86_64/**
third_party/snpe/x86_64-linux-clang/**

Write Preview
Loading…
Cancel
Save