diff --git a/.github/workflows/prebuilt.yaml b/.github/workflows/prebuilt.yaml index 807ac06b94..a6808a0276 100644 --- a/.github/workflows/prebuilt.yaml +++ b/.github/workflows/prebuilt.yaml @@ -30,7 +30,7 @@ jobs: ref: master wait-interval: 30 running-workflow-name: 'build prebuilt' - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: true - name: Build Docker image diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 98cdeab306..144718df60 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -26,10 +26,11 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 50 env: - STRIPPED_DIR: tmppilot + STRIPPED_DIR: /tmp/releasepilot steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: + fetch-depth: 0 submodules: true - name: Check submodules if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot' @@ -46,16 +47,10 @@ jobs: restore-keys: | scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- scons- - - name: Strip non-release files + - name: Build devel run: | - mkdir $STRIPPED_DIR - cp -pR --parents $(cat release/files_common) $STRIPPED_DIR - cp -pR --parents $(cat release/files_tici) $STRIPPED_DIR - cp -pR --parents $(cat release/files_pc) $STRIPPED_DIR + TARGET_DIR=$STRIPPED_DIR release/build_devel.sh cp Dockerfile.openpilot_base $STRIPPED_DIR - - # need this to build on x86 - cp -pR --parents third_party/libyuv third_party/snpe selfdrive/modeld/runners $STRIPPED_DIR - name: Build Docker image run: eval "$BUILD" - name: Build openpilot and run checks @@ -63,32 +58,43 @@ jobs: cd $STRIPPED_DIR ${{ env.RUN }} "CI=1 python selfdrive/manager/build.py && \ python -m unittest discover selfdrive/car" - - name: Cleanup scons cache - run: | - cd $STRIPPED_DIR - ${{ env.RUN }} "scons -j$(nproc) && \ - rm -rf /tmp/scons_cache/* && \ - scons -j$(nproc) --cache-populate" build_all: name: build all runs-on: ubuntu-20.04 timeout-minutes: 50 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: true + - name: Cache scons + id: scons-cache + # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. + uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b + env: + CACHE_SKIP_SAVE: ${{ github.ref != 'refs/heads/master' || github.repository != 'commaai/openpilot' }} + with: + path: /tmp/scons_cache + key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-${{ steps.date.outputs.time }} + restore-keys: | + scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- + scons- - name: Build Docker image run: eval "$BUILD" - name: Build openpilot with all flags run: ${{ env.RUN }} "scons -j$(nproc) --extras --test" + - name: Cleanup scons cache + run: | + ${{ env.RUN }} "scons -j$(nproc) --extras --test && \ + rm -rf /tmp/scons_cache/* && \ + scons -j$(nproc) --cache-populate" #build_mac: # name: build macos # runs-on: macos-latest # timeout-minutes: 60 # steps: - # - uses: actions/checkout@v2 + # - uses: actions/checkout@v3 # with: # submodules: true # - name: Determine pre-existing Homebrew packages @@ -147,9 +153,21 @@ jobs: env: IMAGE_NAME: openpilotwebcamci steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: true + - name: Cache scons + id: scons-cache + # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged. + uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b + env: + CACHE_SKIP_SAVE: true + with: + path: /tmp/scons_cache + key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- + restore-keys: | + scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}- + scons- - name: Build Docker image run: | eval "$BUILD" @@ -170,7 +188,7 @@ jobs: if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' needs: static_analysis # hack to ensure slow tests run first since this and static_analysis are fast steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: true - name: Build Docker image @@ -185,20 +203,20 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 50 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: true - name: Build Docker image run: eval "$BUILD" - name: pre-commit - run: ${{ env.RUN }} "git init && git add -A && pre-commit run --all" + run: ${{ env.RUN }} "pre-commit run --all" valgrind: name: valgrind runs-on: ubuntu-20.04 timeout-minutes: 50 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: true - name: Cache dependencies @@ -241,7 +259,7 @@ jobs: run: echo $TIMESTAMP env: TIMESTAMP: ${{ steps.date.outputs.time }} - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: true - name: Cache scons @@ -289,7 +307,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 50 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: true - name: Cache dependencies @@ -334,27 +352,12 @@ jobs: - name: "Upload coverage to Codecov" uses: codecov/codecov-action@v2 - #model_replay: - # name: model replay - # runs-on: ubuntu-20.04 - # timeout-minutes: 50 - # steps: - # - uses: actions/checkout@v2 - # with: - # submodules: true - # - name: Build Docker image - # run: eval "$BUILD" - # - name: Run replay - # run: | - # ${{ env.RUN }} "scons -j$(nproc) && \ - # selfdrive/test/process_replay/model_replay.py" - test_longitudinal: name: longitudinal runs-on: ubuntu-20.04 timeout-minutes: 50 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: true - name: Cache scons @@ -394,9 +397,9 @@ jobs: strategy: fail-fast: false matrix: - job: [0, 1, 2, 3] + job: [0, 1, 2, 3, 4] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: true - name: Cache dependencies @@ -426,7 +429,7 @@ jobs: coverage xml && \ chmod -R 777 /tmp/comma_download_cache" env: - NUM_JOBS: 4 + NUM_JOBS: 5 JOB_ID: ${{ matrix.job }} - name: "Upload coverage to Codecov" uses: codecov/codecov-action@v2 @@ -436,7 +439,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 50 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: true - name: Build docker container diff --git a/.github/workflows/tools_tests.yaml b/.github/workflows/tools_tests.yaml index 0b0df4c516..2c2bbd30bd 100644 --- a/.github/workflows/tools_tests.yaml +++ b/.github/workflows/tools_tests.yaml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-20.04 timeout-minutes: 30 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: submodules: true - name: Build Docker image @@ -42,28 +42,16 @@ jobs: IMAGE_NAME: openpilot-sim if: github.repository == 'commaai/openpilot' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: + lfs: true submodules: true - - # HACK: cache LFS objects since they count against our quota - # https://github.com/actions/checkout/issues/165#issuecomment-657673315 - - name: Create LFS file list - run: git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id - - name: Restore LFS cache - uses: actions/cache@v2 - id: lfs-cache - with: - path: .git/lfs - key: ${{ runner.os }}-lfs-${{ hashFiles('.lfs-assets-id') }} - - name: Git LFS Pull - run: git lfs pull - - - name: Build Docker image - run: | - eval "$BUILD" - docker pull $DOCKER_REGISTRY/$IMAGE_NAME:latest || true - docker build --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/sim/Dockerfile.sim . + - name: Build base image + run: eval "$BUILD" + - name: Pull latest simulator image + run: eval "$BUILD"docker pull $DOCKER_REGISTRY/$IMAGE_NAME:latest || true + - name: Build simulator image + run: docker build --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/sim/Dockerfile.sim . - name: Push to container registry if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot' run: | diff --git a/Jenkinsfile b/Jenkinsfile index cc7347b12a..3b134d7c0d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -80,27 +80,6 @@ pipeline { stages { stage('parallel tests') { parallel { - /* - stage('Power Consumption Tests') { - steps { - lock(resource: "", label: "c2-zookeeper", inversePrecedence: true, variable: 'device_ip', quantity: 1) { - timeout(time: 90, unit: 'MINUTES') { - sh script: "/home/batman/tools/zookeeper/enable_and_wait.py $device_ip 120", label: "turn on device" - phone(device_ip, "git checkout", readFile("selfdrive/test/setup_device_ci.sh"),) - phone(device_ip, "build", "scons -j4 && sync") - sh script: "/home/batman/tools/zookeeper/disable.py $device_ip", label: "turn off device" - sh script: "/home/batman/tools/zookeeper/enable_and_wait.py $device_ip 120", label: "turn on device" - sh script: "/home/batman/tools/zookeeper/check_consumption.py 60 3", label: "idle power consumption after boot" - sh script: "/home/batman/tools/zookeeper/ignition.py 1", label: "go onroad" - sh script: "/home/batman/tools/zookeeper/check_consumption.py 60 10", label: "onroad power consumption" - sh script: "/home/batman/tools/zookeeper/ignition.py 0", label: "go offroad" - sh script: "/home/batman/tools/zookeeper/check_consumption.py 60 2", label: "idle power consumption offroad" - } - } - } - } - */ - stage('build') { environment { R3_PUSH = "${env.BRANCH_NAME == 'master' ? '1' : ' '}" diff --git a/release/files_pc b/release/files_pc index 610516bff6..e401badb80 100644 --- a/release/files_pc +++ b/release/files_pc @@ -1,3 +1,9 @@ +selfdrive/modeld/runners/onnx* + third_party/mapbox-gl-native-qt/x86_64/*.so third_party/qt-plugins/x86_64/geoservices/*.so + +third_party/libyuv/x64/** +third_party/snpe/x86_64/** +third_party/snpe/x86_64-linux-clang/**