diff --git a/.devcontainer/.gitignore b/.devcontainer/.gitignore deleted file mode 100644 index 5682fe1190..0000000000 --- a/.devcontainer/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -.Xauthority -.env -.host/ \ No newline at end of file diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index af67bdbe5d..0000000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM ghcr.io/commaai/openpilot-base:latest - -RUN apt update && apt install -y vim net-tools usbutils htop ripgrep tmux wget mesa-utils xvfb libxtst6 libxv1 libglu1-mesa libegl1-mesa gdb bash-completion -RUN pip install ipython jupyter jupyterlab - -RUN cd /tmp && \ - ARCH=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && \ - curl -L -o virtualgl.deb "https://downloads.sourceforge.net/project/virtualgl/3.1/virtualgl_3.1_$ARCH.deb" && \ - dpkg -i virtualgl.deb - -RUN usermod -aG video batman - -USER batman - -RUN cd $HOME && \ - curl -O https://raw.githubusercontent.com/commaai/agnos-builder/master/userspace/home/.tmux.conf && \ - curl -O https://raw.githubusercontent.com/commaai/agnos-builder/master/userspace/home/.vimrc diff --git a/.devcontainer/container_post_create.sh b/.devcontainer/container_post_create.sh deleted file mode 100755 index dee1c61101..0000000000 --- a/.devcontainer/container_post_create.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash - -TARGET_USER=batman -source .devcontainer/.host/.env - -# override display flag for mac hosts -if [[ $HOST_OS == darwin ]]; then - echo "Setting up DISPLAY override for macOS..." - cat <> /home/$TARGET_USER/.bashrc -source .devcontainer/.host/.env -if [ -n "\$HOST_DISPLAY" ]; then - DISPLAY_NUM=\$(echo "\$HOST_DISPLAY" | awk -F: '{print \$NF}') - export DISPLAY=host.docker.internal:\$DISPLAY_NUM -fi -EOF -fi - -# setup virtualgl for mac hosts -if [[ $HOST_OS == darwin ]]; then - echo "Setting up virtualgl for macOS..." - cat <> /home/$TARGET_USER/.bashrc -if [ -n "\$HOST_DISPLAY" ]; then - export VGL_PORT=10000 - export VGL_CLIENT=host.docker.internal - export VGL_COMPRESS=rgb - export VGL_DISPLAY=:99 - export VGL_FPS=60 - # prevent vglrun from running exec - alias exec=:; source vglrun :; unalias exec -fi -EOF -fi - -# These lines are temporary, to remain backwards compatible with old devcontainers -# that were running as root and therefore had their caches written as root -sudo chown -R $TARGET_USER: /tmp/scons_cache -sudo chown -R $TARGET_USER: /tmp/comma_download_cache -sudo chown -R $TARGET_USER: /home/batman/.comma diff --git a/.devcontainer/container_post_start.sh b/.devcontainer/container_post_start.sh deleted file mode 100755 index 1521b9c3fb..0000000000 --- a/.devcontainer/container_post_start.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -source .devcontainer/.host/.env - -# setup safe directories for submodules -SUBMODULE_DIRS=$(git config --file .gitmodules --get-regexp path | awk '{ print $2 }') -for DIR in $SUBMODULE_DIRS; do - git config --global --add safe.directory "$PWD/$DIR" -done - -# virtual display for virtualgl -if [[ "$HOST_OS" == "darwin" ]] && [[ -n "$HOST_DISPLAY" ]]; then - echo "Starting virtual display at :99 ..." - tmux new-session -d -s fakedisplay Xvfb :99 -screen 0 1920x1080x24 -fi diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json deleted file mode 100644 index 3f5b38d12e..0000000000 --- a/.devcontainer/devcontainer.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "name": "openpilot devcontainer", - "build": { - "dockerfile": "Dockerfile" - }, - "postCreateCommand": ".devcontainer/container_post_create.sh", - "postStartCommand": ".devcontainer/container_post_start.sh", - "initializeCommand": [".devcontainer/host_setup"], - "privileged": true, - "containerEnv": { - "DISPLAY": "${localEnv:DISPLAY}", - "PYTHONPATH": "${containerWorkspaceFolder}", - "TERM": "xterm-256color", - "force_color_prompt": "1" - }, - "runArgs": [ - "--volume=/dev:/dev", - "--volume=/tmp/.X11-unix:/tmp/.X11-unix", - "--volume=${localWorkspaceFolder}/.devcontainer/.host/.Xauthority:/home/batman/.Xauthority", - "--volume=${localEnv:HOME}/.comma:/home/batman/.comma", - "--volume=${localEnv:HOME}/.azure:/home/batman/.azure", - "--volume=/tmp/comma_download_cache:/tmp/comma_download_cache", - "--shm-size=1G", - "--add-host=host.docker.internal:host-gateway", // required to use host.docker.internal on linux - "--publish=0.0.0.0:8070-8079:8070-8079" // body ZMQ services - ], - "features": { - "ghcr.io/devcontainers/features/common-utils:2": { - "installZsh": false, - "installOhMyZsh": false, - "upgradePackages": false, - "username": "batman" - }, - "ghcr.io/devcontainers-contrib/features/gh-cli:1": {}, - "ghcr.io/devcontainers/features/azure-cli:1": {} - }, - "containerUser": "batman", - "remoteUser": "batman", - "customizations": { - "vscode": { - "extensions": [ - "ms-python.python", - "ms-vscode.cpptools", - "ms-toolsai.jupyter", - "guyskk.language-cython", - "lharri73.dbc" - ] - } - }, - "mounts": [ - "type=volume,source=scons_cache,target=/tmp/scons_cache" - ] -} diff --git a/.devcontainer/host_setup b/.devcontainer/host_setup deleted file mode 100755 index 8ff81ebe42..0000000000 --- a/.devcontainer/host_setup +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -# pull base image -if [[ -z $USE_LOCAL_IMAGE ]]; then - echo "Updating openpilot_base image if needed..." - docker pull ghcr.io/commaai/openpilot-base:latest -fi - -# setup .host dir -mkdir -p .devcontainer/.host - -# setup links to Xauthority -XAUTHORITY_LINK=".devcontainer/.host/.Xauthority" -rm -f $XAUTHORITY_LINK -if [[ -z $XAUTHORITY ]]; then - echo "XAUTHORITY not set. Fallback to ~/.Xauthority ..." - if ! [[ -f $HOME/.Xauthority ]]; then - echo "~/.XAuthority file does not exist. GUI tools may not work properly." - touch $XAUTHORITY_LINK # dummy file to satisfy container volume mount - else - ln -sf $HOME/.Xauthority $XAUTHORITY_LINK - fi -else - ln -sf $XAUTHORITY $XAUTHORITY_LINK -fi - -# setup host env file -HOST_INFO_FILE=".devcontainer/.host/.env" -SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') -echo "HOST_OS=\"$SYSTEM\"" > $HOST_INFO_FILE -echo "HOST_DISPLAY=\"$DISPLAY\"" >> $HOST_INFO_FILE - -# run virtualgl if macos -if [[ $SYSTEM == "darwin" ]]; then - echo - if [[ -f /opt/VirtualGL/bin/vglclient ]]; then - echo "Starting VirtualGL client at port 10000..." - VGL_LOG_FILE=".devcontainer/.host/.vgl/vglclient.log" - mkdir -p "$(dirname $VGL_LOG_FILE)" - /opt/VirtualGL/bin/vglclient -l "$VGL_LOG_FILE" -display "$DISPLAY" -port 10000 -detach - else - echo "VirtualGL not found. GUI tools may not work properly. Some GUI tools require OpenGL to work properly. To use them with XQuartz on mac, VirtualGL needs to be installed. To install it run:" - echo - echo " brew install --cask virtualgl" - echo - fi -fi diff --git a/.devcontainer/host_setup.cmd b/.devcontainer/host_setup.cmd deleted file mode 100644 index 885bd27f25..0000000000 --- a/.devcontainer/host_setup.cmd +++ /dev/null @@ -1,10 +0,0 @@ -:: pull base image -IF NOT DEFINED USE_LOCAL_IMAGE ^ -echo "Updating openpilot_base image if needed..." && ^ -docker pull ghcr.io/commaai/openpilot-base:latest - -:: setup .host dir -mkdir .devcontainer\.host - -:: setup host env file -echo "" > .devcontainer\.host\.env \ No newline at end of file diff --git a/.gitattributes b/.gitattributes index 3a1f47eda0..eda2505d0e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,13 +2,13 @@ # to move existing files into LFS: # git add --renormalize . -*.dlc filter=lfs diff=lfs merge=lfs -text *.onnx filter=lfs diff=lfs merge=lfs -text *.svg filter=lfs diff=lfs merge=lfs -text *.png filter=lfs diff=lfs merge=lfs -text *.gif filter=lfs diff=lfs merge=lfs -text *.ttf filter=lfs diff=lfs merge=lfs -text *.wav filter=lfs diff=lfs merge=lfs -text + selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text system/hardware/tici/updater filter=lfs diff=lfs merge=lfs -text selfdrive/ui/qt/spinner_larch64 filter=lfs diff=lfs merge=lfs -text @@ -21,72 +21,3 @@ third_party/acados/*/t_renderer filter=lfs diff=lfs merge=lfs -text third_party/qt5/larch64/bin/lrelease filter=lfs diff=lfs merge=lfs -text third_party/qt5/larch64/bin/lupdate filter=lfs diff=lfs merge=lfs -text third_party/catch2/include/catch2/catch.hpp filter=lfs diff=lfs merge=lfs -text -*.ico filter=lfs diff=lfs merge=lfs -text -*.apk filter=lfs diff=lfs merge=lfs -text -*.apkpatch filter=lfs diff=lfs merge=lfs -text -*.jar filter=lfs diff=lfs merge=lfs -text -*.pdf filter=lfs diff=lfs merge=lfs -text -*.jpg filter=lfs diff=lfs merge=lfs -text -*.mp3 filter=lfs diff=lfs merge=lfs -text -*.thneed filter=lfs diff=lfs merge=lfs -text -*.tar.gz filter=lfs diff=lfs merge=lfs -text -*.npy filter=lfs diff=lfs merge=lfs -text -*.csv filter=lfs diff=lfs merge=lfs -text -*.a filter=lfs diff=lfs merge=lfs -text -*.so* filter=lfs diff=lfs merge=lfs -text -*.dylib filter=lfs diff=lfs merge=lfs -text -*.o filter=lfs diff=lfs merge=lfs -text -*.b64 filter=lfs diff=lfs merge=lfs -text -selfdrive/hardware/tici/updater filter=lfs diff=lfs merge=lfs -text -selfdrive/boardd/tests/test_boardd filter=lfs diff=lfs merge=lfs -text -selfdrive/ui/qt/spinner_aarch64 filter=lfs diff=lfs merge=lfs -text -installer/updater/updater filter=lfs diff=lfs merge=lfs -text -selfdrive/debug/profiling/simpleperf/**/* filter=lfs diff=lfs merge=lfs -text -selfdrive/hardware/eon/updater filter=lfs diff=lfs merge=lfs -text -selfdrive/ui/qt/text_aarch64 filter=lfs diff=lfs merge=lfs -text -selfdrive/debug/profiling/pyflame/**/* filter=lfs diff=lfs merge=lfs -text -installer/installers/installer_openpilot filter=lfs diff=lfs merge=lfs -text -installer/installers/installer_dashcam filter=lfs diff=lfs merge=lfs -text -selfdrive/ui/text/text filter=lfs diff=lfs merge=lfs -text -selfdrive/ui/android/text/text filter=lfs diff=lfs merge=lfs -text -selfdrive/ui/spinner/spinner filter=lfs diff=lfs merge=lfs -text -selfdrive/visiond/visiond filter=lfs diff=lfs merge=lfs -text -selfdrive/loggerd/loggerd filter=lfs diff=lfs merge=lfs -text -selfdrive/sensord/sensord filter=lfs diff=lfs merge=lfs -text -selfdrive/sensord/gpsd filter=lfs diff=lfs merge=lfs -text -selfdrive/ui/android/spinner/spinner filter=lfs diff=lfs merge=lfs -text -selfdrive/ui/qt/spinner filter=lfs diff=lfs merge=lfs -text -selfdrive/ui/qt/text filter=lfs diff=lfs merge=lfs -text -_stringdefs.py filter=lfs diff=lfs merge=lfs -text -dfu-util-aarch64-linux filter=lfs diff=lfs merge=lfs -text -dfu-util-aarch64 filter=lfs diff=lfs merge=lfs -text -dfu-util-x86_64-linux filter=lfs diff=lfs merge=lfs -text -dfu-util-x86_64 filter=lfs diff=lfs merge=lfs -text -stb_image.h filter=lfs diff=lfs merge=lfs -text -clpeak3 filter=lfs diff=lfs merge=lfs -text -clwaste filter=lfs diff=lfs merge=lfs -text -apk/**/* filter=lfs diff=lfs merge=lfs -text -external/**/* filter=lfs diff=lfs merge=lfs -text -phonelibs/**/* filter=lfs diff=lfs merge=lfs -text -third_party/boringssl/**/* filter=lfs diff=lfs merge=lfs -text -pyextra/**/* filter=lfs diff=lfs merge=lfs -text -panda/board/**/inc/*.h filter=lfs diff=lfs merge=lfs -text -panda/board/obj/*.elf filter=lfs diff=lfs merge=lfs -text -board/inc/*.h filter=lfs diff=lfs merge=lfs -text -third_party/nanovg/**/* filter=lfs diff=lfs merge=lfs -text -selfdrive/controls/lib/lateral_mpc/lib_mpc_export/**/* filter=lfs diff=lfs merge=lfs -text -third_party/android_hardware_libhardware/**/* filter=lfs diff=lfs merge=lfs -text -selfdrive/controls/lib/lead_mpc_lib/lib_mpc_export/**/* filter=lfs diff=lfs merge=lfs -text -*.pro filter=lfs diff=lfs merge=lfs -text -selfdrive/controls/lib/longitudinal_mpc/lib_mpc_export/**/* filter=lfs diff=lfs merge=lfs -text -selfdrive/controls/lib/lateral_mpc/mpc_export/**/* filter=lfs diff=lfs merge=lfs -text -third_party/curl/**/* filter=lfs diff=lfs merge=lfs -text -selfdrive/modeld/thneed/debug/**/* filter=lfs diff=lfs merge=lfs -text -selfdrive/modeld/thneed/include/**/* filter=lfs diff=lfs merge=lfs -text -third_party/openmax/**/* filter=lfs diff=lfs merge=lfs -text -selfdrive/controls/lib/longitudinal_mpc/mpc_export/**/* filter=lfs diff=lfs merge=lfs -text -selfdrive/controls/lib/longitudinal_mpc_model/lib_mpc_export/**/* filter=lfs diff=lfs merge=lfs -text -Pipfile filter=lfs diff=lfs merge=lfs -text -Pipfile.lock filter=lfs diff=lfs merge=lfs -text -poetry.lock filter=lfs diff=lfs merge=lfs -text -*.qm filter=lfs diff=lfs merge=lfs -text diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 45a8af0aaf..9e9f6af61d 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,14 +1,11 @@ blank_issues_enabled: false contact_links: + - name: Join the Discord + url: https://discord.comma.ai + about: The community Discord is for both openpilot development and experience discussion - name: Report model bugs - url: https://github.com/commaai/openpilot/discussions/categories/model-feedback - about: Provide feedback for the driving or driver monitoring models - - name: Discussions - url: https://github.com/commaai/openpilot/discussions - about: For questions and general discussion about openpilot + url: https://discord.com/channels/469524606043160576/1254834193066623017 + about: Feedback for the driving and driver monitoring models goes in the #driving-feedback in Discord - name: Community Wiki url: https://github.com/commaai/openpilot/wiki about: Check out our community wiki - - name: Community Discord - url: https://discord.comma.ai - about: Check out our community discord diff --git a/.github/PULL_REQUEST_TEMPLATE/bugfix.md b/.github/PULL_REQUEST_TEMPLATE/bugfix.md deleted file mode 100644 index e28661db3b..0000000000 --- a/.github/PULL_REQUEST_TEMPLATE/bugfix.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -name: Bug fix -about: For openpilot bug fixes -title: '' -labels: 'bugfix' -assignees: '' ---- - -**Description** - - - -**Verification** - - diff --git a/.github/PULL_REQUEST_TEMPLATE/car_bugfix.md b/.github/PULL_REQUEST_TEMPLATE/car_bugfix.md deleted file mode 100644 index 76c86346c8..0000000000 --- a/.github/PULL_REQUEST_TEMPLATE/car_bugfix.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -name: Car Bug fix -about: For vehicle/brand specific bug fixes -title: '' -labels: 'car bug fix' -assignees: '' ---- - -**Description** - - - -**Verification** - - - -**Route** - -Route: [a route with the bug fix] diff --git a/.github/PULL_REQUEST_TEMPLATE/car_port.md b/.github/PULL_REQUEST_TEMPLATE/car_port.md deleted file mode 100644 index c7aa2b96c2..0000000000 --- a/.github/PULL_REQUEST_TEMPLATE/car_port.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -name: Car port -about: For new car ports -title: '' -labels: 'car port' -assignees: '' ---- - -**Checklist** - -- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs -- [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py) -- [ ] route with openpilot: -- [ ] route with stock system: -- [ ] car harness used (if comma doesn't sell it, put N/A): diff --git a/.github/PULL_REQUEST_TEMPLATE/fingerprint.md b/.github/PULL_REQUEST_TEMPLATE/fingerprint.md deleted file mode 100644 index b94f7dc53f..0000000000 --- a/.github/PULL_REQUEST_TEMPLATE/fingerprint.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -name: Fingerprint -about: For adding fingerprints to existing cars -title: '' -labels: 'fingerprint' -assignees: '' ---- - -**Car** -Which car (make, model, year) this fingerprint is for - -**Route** -A route with the fingerprint \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE/refactor.md b/.github/PULL_REQUEST_TEMPLATE/refactor.md deleted file mode 100644 index 1ee21c1bba..0000000000 --- a/.github/PULL_REQUEST_TEMPLATE/refactor.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -name: Refactor -about: For code refactors -title: '' -labels: 'refactor' -assignees: '' ---- - -**Description** - - - -**Verification** - - diff --git a/.github/PULL_REQUEST_TEMPLATE/tuning.md b/.github/PULL_REQUEST_TEMPLATE/tuning.md deleted file mode 100644 index 4397e5ad20..0000000000 --- a/.github/PULL_REQUEST_TEMPLATE/tuning.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -name: Tuning -about: For openpilot tuning changes -title: '' -labels: 'tuning' -assignees: '' ---- - -**Description** - - - -**Verification** - - \ No newline at end of file diff --git a/.github/build.py b/.github/build.py deleted file mode 100644 index e141ea05df..0000000000 --- a/.github/build.py +++ /dev/null @@ -1,30 +0,0 @@ -import pathlib - -GITHUB_FOLDER = pathlib.Path(__file__).parent - -PULL_REQUEST_TEMPLATES = (GITHUB_FOLDER / "PULL_REQUEST_TEMPLATE") - -order = ["fingerprint", "car_bugfix", "bugfix", "car_port", "refactor"] - -def create_pull_request_template(): - with open(GITHUB_FOLDER / "pull_request_template.md", "w") as f: - f.write("\n\n") - - for t in order: - template = PULL_REQUEST_TEMPLATES / f"{t}.md" - text = template.read_text() - - # Remove metadata for GitHub - start = text.find("---") - end = text.find("---", start+1) - text = text[end + 4:] - - # Remove comments - text = text.replace("", "") - - f.write(f"\n\n") - -create_pull_request_template() diff --git a/.github/labeler.yaml b/.github/labeler.yaml index 47d0d480a9..db1f976da8 100644 --- a/.github/labeler.yaml +++ b/.github/labeler.yaml @@ -4,59 +4,7 @@ CI / testing: car: - changed-files: - - any-glob-to-all-files: 'selfdrive/car/**' - -body: - - changed-files: - - any-glob-to-all-files: 'selfdrive/car/body/*' - -chrysler: - - changed-files: - - any-glob-to-all-files: 'selfdrive/car/chrysler/*' - -ford: - - changed-files: - - any-glob-to-all-files: 'selfdrive/car/ford/*' - -gm: - - changed-files: - - any-glob-to-all-files: 'selfdrive/car/gm/*' - -honda: - - changed-files: - - any-glob-to-all-files: 'selfdrive/car/honda/*' - -hyundai: - - changed-files: - - any-glob-to-all-files: 'selfdrive/car/hyundai/*' - -mazda: - - changed-files: - - any-glob-to-all-files: 'selfdrive/car/mazda/*' - -nissan: - - changed-files: - - any-glob-to-all-files: 'selfdrive/car/nissan/*' - -subaru: - - changed-files: - - any-glob-to-all-files: 'selfdrive/car/subaru/*' - -tesla: - - changed-files: - - any-glob-to-all-files: 'selfdrive/car/telsa/*' - -toyota: - - changed-files: - - any-glob-to-all-files: 'selfdrive/car/toyota/*' - -volkswagen: - - changed-files: - - any-glob-to-all-files: 'selfdrive/car/volkswagen/*' - -fingerprint: - - changed-files: - - any-glob-to-all-files: 'selfdrive/car/*/fingerprints.py' + - any-glob-to-all-files: '{selfdrive/car/**,opendbc_repo}' simulation: - changed-files: @@ -74,6 +22,6 @@ multilanguage: - changed-files: - any-glob-to-all-files: 'selfdrive/ui/translations/**' -research: +autonomy: - changed-files: - any-glob-to-all-files: "{selfdrive/modeld/models/**,selfdrive/test/process_replay/model_replay_ref_commit}" diff --git a/.github/workflows/auto-cache/action.yaml b/.github/workflows/auto-cache/action.yaml index e99c46e3c3..fadd422b6e 100644 --- a/.github/workflows/auto-cache/action.yaml +++ b/.github/workflows/auto-cache/action.yaml @@ -14,17 +14,25 @@ inputs: description: 'whether to save the cache' default: 'false' required: false +outputs: + cache-hit: + description: 'cache hit occurred' + value: ${{ (contains(runner.name, 'nsc') && steps.ns-cache.outputs.cache-hit) || + (!contains(runner.name, 'nsc') && inputs.save != 'false' && steps.gha-cache.outputs.cache-hit) || + (!contains(runner.name, 'nsc') && inputs.save == 'false' && steps.gha-cache-ro.outputs.cache-hit) }} runs: using: "composite" steps: - name: setup namespace cache + id: ns-cache if: ${{ contains(runner.name, 'nsc') }} uses: namespacelabs/nscloud-cache-action@v1 with: path: ${{ inputs.path }} - name: setup github cache + id: gha-cache if: ${{ !contains(runner.name, 'nsc') && inputs.save != 'false' }} uses: 'actions/cache@v4' with: @@ -33,6 +41,7 @@ runs: restore-keys: ${{ inputs.restore-keys }} - name: setup github cache + id: gha-cache-ro if: ${{ !contains(runner.name, 'nsc') && inputs.save == 'false' }} uses: 'actions/cache/restore@v4' with: diff --git a/.github/workflows/auto_pr_review.yaml b/.github/workflows/auto_pr_review.yaml index 447e559c90..eaaca56060 100644 --- a/.github/workflows/auto_pr_review.yaml +++ b/.github/workflows/auto_pr_review.yaml @@ -35,11 +35,12 @@ jobs: already-exists-comment: "Your PR should be made against the `master` branch" # Welcome comment - - name: comment - uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6 + - name: "First timers PR" + uses: actions/first-interaction@v1 if: github.event.pull_request.head.repo.full_name != 'commaai/openpilot' with: - message: | + repo-token: ${{ secrets.GITHUB_TOKEN }} + pr-message: | Thanks for contributing to openpilot! In order for us to review your PR as quickly as possible, check the following: * Convert your PR to a draft unless it's ready to review @@ -49,5 +50,4 @@ jobs: * all the tests are passing * the change is [something we merge](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md#what-gets-merged) * include a route or your device' dongle ID if relevant - comment_tag: run_id - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + diff --git a/.github/workflows/badges.yaml b/.github/workflows/badges.yaml index cf8bdc7109..63ee736dca 100644 --- a/.github/workflows/badges.yaml +++ b/.github/workflows/badges.yaml @@ -7,7 +7,7 @@ on: env: BASE_IMAGE: openpilot-base DOCKER_REGISTRY: ghcr.io/commaai - RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c + RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c jobs: badges: @@ -23,7 +23,7 @@ jobs: - uses: ./.github/workflows/setup-with-retry - name: Push badges run: | - ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/ui/translations/create_badges.py" + ${{ env.RUN }} "scons -j$(nproc) && python3 selfdrive/ui/translations/create_badges.py" rm .gitattributes diff --git a/.github/workflows/ci_weekly_report.yaml b/.github/workflows/ci_weekly_report.yaml new file mode 100644 index 0000000000..9821283cb5 --- /dev/null +++ b/.github/workflows/ci_weekly_report.yaml @@ -0,0 +1,101 @@ +name: weekly CI test report +on: + schedule: + - cron: '37 9 * * 1' # 9:37AM UTC -> 2:37AM PST every monday + workflow_dispatch: + inputs: + ci_runs: + description: 'The amount of runs to trigger in CI test report' +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CI_RUNS: ${{ github.event.inputs.ci_runs || '50' }} + +jobs: + setup: + if: github.repository == 'commaai/openpilot' + runs-on: ubuntu-latest + outputs: + ci_runs: ${{ steps.ci_runs_setup.outputs.matrix }} + steps: + - id: ci_runs_setup + name: CI_RUNS=${{ env.CI_RUNS }} + run: | + matrix=$(python3 -c "import json; print(json.dumps({ 'run_number' : list(range(${{ env.CI_RUNS }})) }))") + echo "matrix=$matrix" >> $GITHUB_OUTPUT + + ci_matrix_run: + needs: [ setup ] + strategy: + fail-fast: false + matrix: ${{fromJSON(needs.setup.outputs.ci_runs)}} + uses: commaai/openpilot/.github/workflows/ci_weekly_run.yaml@master + with: + run_number: ${{ matrix.run_number }} + + report: + needs: [ci_matrix_run] + runs-on: ubuntu-latest + if: always() + steps: + - name: Get job results + uses: actions/github-script@v7 + id: get-job-results + with: + script: | + const jobs = await github + .paginate("GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs", { + owner: "commaai", + repo: "${{ github.event.repository.name }}", + run_id: "${{ github.run_id }}", + attempt: "${{ github.run_attempt }}", + }) + var report = {} + jobs.slice(1, jobs.length-1).forEach(job => { + if (job.conclusion === "skipped") return; + const jobName = job.name.split(" / ")[2]; + const runRegex = /\((.*?)\)/; + const run = job.name.match(runRegex)[1]; + report[jobName] = report[jobName] || { successes: [], failures: [], canceled: [] }; + switch (job.conclusion) { + case "success": + report[jobName].successes.push({ "run_number": run, "link": job.html_url}); break; + case "failure": + report[jobName].failures.push({ "run_number": run, "link": job.html_url }); break; + case "canceled": + report[jobName].canceled.push({ "run_number": run, "link": job.html_url }); break; + } + }); + return JSON.stringify({"jobs": report}); + + - name: Add job results to summary + env: + JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }} + run: | + cat <> template.html + + + + + + + + + + + {% for key in jobs.keys() %} + + + + + + {% endfor %} +
Job✅ Passing❌ Failure Details
{% for i in range(5) %}{% if i+1 <= (5 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }}) %}🟩{% else %}🟥{% endif %}{% endfor%}{{ key }}{{ 100 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }} }}%{% if jobs[key]["failures"]|length > 0 %}
{% for failure in jobs[key]["failures"] %}Log for run #{{ failure['run_number'] }}
{% endfor %}
{% else %}{% endif %}
+ EOF + + pip install jinja2-cli + echo $JOB_RESULTS | jinja2 template.html > report.html + echo "# CI Test Report - ${{ env.CI_RUNS }} Runs" >> $GITHUB_STEP_SUMMARY + cat report.html >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/ci_weekly_run.yaml b/.github/workflows/ci_weekly_run.yaml new file mode 100644 index 0000000000..d8bf91116d --- /dev/null +++ b/.github/workflows/ci_weekly_run.yaml @@ -0,0 +1,17 @@ +name: weekly CI test run +on: + workflow_call: + inputs: + run_number: + required: true + type: string + +concurrency: + group: ci-run-${{ inputs.run_number }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + selfdrive_tests: + uses: commaai/openpilot/.github/workflows/selfdrive_tests.yaml@master + with: + run_number: ${{ inputs.run_number }} diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index a32989f7d0..92c311829c 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -5,35 +5,35 @@ on: branches: - master pull_request: - + workflow_call: + inputs: + run_number: + default: '1' + required: true + type: string concurrency: - group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }} + group: docs-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }} cancel-in-progress: true -env: - BASE_IMAGE: openpilot-base - - BUILD: selfdrive/test/docker_build.sh base - - RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c - jobs: docs: name: build docs - runs-on: ubuntu-latest - timeout-minutes: 45 + runs-on: ubuntu-24.04 steps: + - uses: commaai/timeout@v1 + - uses: actions/checkout@v4 with: submodules: true - - uses: ./.github/workflows/setup-with-retry - - name: Build openpilot - run: | - ${{ env.RUN }} "scons -j$(nproc)" + + # Build - name: Build docs run: | - ${{ env.RUN }} "apt update && apt install -y doxygen && cd docs && make -j$(nproc) html" + # TODO: can we install just the "docs" dependency group without the normal deps? + pip install mkdocs + mkdocs build + # Push to docs.comma.ai - uses: actions/checkout@v4 if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot' with: @@ -48,16 +48,17 @@ jobs: source release/identity.sh cd openpilot-docs - git checkout --orphan tmp git rm -rf . - cp -r ../build/docs/html/ docs/ - cp -r ../docs/README.md . + # copy over docs + cp -r ../docs_site/ docs/ + + # GitHub pages config touch docs/.nojekyll echo -n docs.comma.ai > docs/CNAME - git add -f . + git add -f . git commit -m "build docs" # docs live in different repo to not bloat openpilot's full clone size diff --git a/.github/workflows/jenkins-pr-trigger.yaml b/.github/workflows/jenkins-pr-trigger.yaml new file mode 100644 index 0000000000..db1d524018 --- /dev/null +++ b/.github/workflows/jenkins-pr-trigger.yaml @@ -0,0 +1,45 @@ +name: jenkins scan + +on: + issue_comment: + types: [created, edited] + +jobs: + # TODO: gc old branches in a separate job in this workflow + scan-comments: + runs-on: ubuntu-latest + if: ${{ github.event.issue.pull_request }} + steps: + - name: Check for trigger phrase + id: check_comment + uses: actions/github-script@v7 + with: + script: | + const triggerPhrase = "trigger-jenkins"; + const comment = context.payload.comment.body; + const commenter = context.payload.comment.user.login; + + const { data: permissions } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username: commenter + }); + + const hasWriteAccess = permissions.permission === 'write' || permissions.permission === 'admin'; + + return (hasWriteAccess && comment.includes(triggerPhrase)); + result-encoding: json + + - name: Checkout repository + if: steps.check_comment.outputs.result == 'true' + uses: actions/checkout@v4 + with: + ref: refs/pull/${{ github.event.issue.number }}/head + + - name: Push to tmp-jenkins branch + if: steps.check_comment.outputs.result == 'true' + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git checkout -b tmp-jenkins-${{ github.event.issue.number }} + GIT_LFS_SKIP_PUSH=1 git push -f origin tmp-jenkins-${{ github.event.issue.number }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 34e333bb59..c86c044772 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,7 +1,7 @@ name: release on: schedule: - - cron: '0 10 * * *' + - cron: '0 9 * * *' workflow_dispatch: jobs: diff --git a/.github/workflows/repo-maintenance.yaml b/.github/workflows/repo-maintenance.yaml index 0e6606112b..ab0f3c1bee 100644 --- a/.github/workflows/repo-maintenance.yaml +++ b/.github/workflows/repo-maintenance.yaml @@ -6,8 +6,8 @@ on: workflow_dispatch: jobs: - bump_submodules: - name: bump_submodules + package_updates: + name: package_updates runs-on: ubuntu-latest container: image: ghcr.io/commaai/openpilot-base:latest @@ -16,46 +16,28 @@ jobs: - uses: actions/checkout@v4 with: submodules: true + - name: uv lock + run: | + python3 -m ensurepip --upgrade + pip3 install uv + uv lock --upgrade - name: bump submodules run: | git config --global --add safe.directory '*' git -c submodule."tinygrad".update=none submodule update --remote git add . - - name: Create Pull Request - uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83 - with: - author: Vehicle Researcher - token: ${{ secrets.ACTIONS_CREATE_PR_PAT }} - commit-message: bump submodules - title: '[bot] Bump submodules' - branch: auto-bump-submodules - base: master - delete-branch: true - body: 'Automatic PR from repo-maintenance -> bump_submodules' - labels: bot - package_updates: - name: package_updates - runs-on: ubuntu-latest - container: - image: ghcr.io/commaai/openpilot-base:latest - if: github.repository == 'commaai/openpilot' - steps: - - uses: actions/checkout@v4 - - name: poetry lock + - name: update car docs run: | - pip install poetry - poetry lock - - name: pre-commit autoupdate - run: | - git config --global --add safe.directory '*' - pre-commit autoupdate + scons -j$(nproc) --minimal opendbc_repo + PYTHONPATH=. python selfdrive/car/docs.py + git add docs/CARS.md - name: Create Pull Request uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83 with: author: Vehicle Researcher token: ${{ secrets.ACTIONS_CREATE_PR_PAT }} - commit-message: Update Python packages and pre-commit hooks - title: '[bot] Update Python packages and pre-commit hooks' + commit-message: Update Python packages + title: '[bot] Update Python packages' branch: auto-package-updates base: master delete-branch: true diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 0ccf1844d9..67cbc5f097 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -6,9 +6,15 @@ on: - master pull_request: workflow_dispatch: + workflow_call: + inputs: + run_number: + default: '1' + required: true + type: string concurrency: - group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }} + group: selfdrive-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }} cancel-in-progress: true env: @@ -19,59 +25,51 @@ env: DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} BUILD: selfdrive/test/docker_build.sh base - RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PRE_COMMIT_HOME=/tmp/pre-commit -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/pre-commit:/tmp/pre-commit -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c + RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 -n logical jobs: build_release: name: build release - runs-on: ubuntu-latest + runs-on: + - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }} + - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }} env: STRIPPED_DIR: /tmp/releasepilot steps: - uses: actions/checkout@v4 with: submodules: true - - run: git lfs pull + - name: Getting LFS files + uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e + with: + timeout_minutes: 2 + max_attempts: 3 + command: git lfs pull - name: Build devel timeout-minutes: 1 run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh - - uses: ./.github/workflows/setup-pre-commit - uses: ./.github/workflows/setup-with-retry - name: Check submodules - if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot' - timeout-minutes: 1 + if: github.repository == 'commaai/openpilot' + timeout-minutes: 3 run: release/check-submodules.sh - name: Build openpilot and run checks timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache run: | cd $STRIPPED_DIR - ${{ env.RUN }} "python system/manager/build.py" + ${{ env.RUN }} "python3 system/manager/build.py" - name: Run tests - timeout-minutes: 3 - run: | - cd $STRIPPED_DIR - ${{ env.RUN }} "release/check-dirty.sh && \ - MAX_EXAMPLES=5 $PYTEST -m 'not slow' selfdrive/car" - - name: pre-commit - timeout-minutes: 3 + timeout-minutes: 1 run: | - cd $GITHUB_WORKSPACE - cp .pre-commit-config.yaml $STRIPPED_DIR - cp pyproject.toml $STRIPPED_DIR - cp poetry.lock $STRIPPED_DIR cd $STRIPPED_DIR - ${{ env.RUN }} "unset PYTHONWARNINGS && SKIP=check-added-large-files,check-hooks-apply,check-useless-excludes pre-commit run --all && chmod -R 777 /tmp/pre-commit" + ${{ env.RUN }} "release/check-dirty.sh" build: - strategy: - matrix: - arch: ${{ fromJson( - ((github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && '["x86_64", "aarch64"]' || '["x86_64"]' ) }} - runs-on: ${{ (matrix.arch == 'aarch64') && 'namespace-profile-arm64-2x8' || 'ubuntu-latest' }} + runs-on: + - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }} + - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }} steps: - uses: actions/checkout@v4 with: @@ -80,72 +78,72 @@ jobs: if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' run: | echo "PUSH_IMAGE=true" >> "$GITHUB_ENV" - echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV" $DOCKER_LOGIN - uses: ./.github/workflows/setup-with-retry - with: - docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }} - uses: ./.github/workflows/compile-openpilot - timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache + timeout-minutes: 30 - docker_push_multiarch: - name: docker push multiarch tag - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' - needs: [build] + build_mac: + name: build macOS + runs-on: ${{ github.repository == 'commaai/openpilot' && 'namespace-profile-macos-8x14' || 'macos-latest' }} steps: - uses: actions/checkout@v4 with: - submodules: false - - name: Setup docker - run: | - $DOCKER_LOGIN - - name: Merge x64 and arm64 tags - run: | - export PUSH_IMAGE=true - scripts/retry.sh selfdrive/test/docker_tag_multiarch.sh base x86_64 aarch64 + submodules: true + - name: Homebrew cache + uses: ./.github/workflows/auto-cache + with: + path: ~/Library/Caches/Homebrew + - name: Install dependencies + run: ./tools/mac_setup.sh + env: + # package install has DeprecationWarnings + PYTHONWARNINGS: default + - run: git lfs pull + - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV + - name: Getting scons cache + uses: ./.github/workflows/auto-cache + with: + path: /tmp/scons_cache + - name: Building openpilot + run: . .venv/bin/activate && scons -j$(nproc) static_analysis: name: static analysis - runs-on: ${{ ((github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }} + runs-on: + - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }} + - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }} + env: + PYTHONWARNINGS: default steps: - uses: actions/checkout@v4 with: submodules: true - - uses: ./.github/workflows/setup-pre-commit - uses: ./.github/workflows/setup-with-retry - - name: Build openpilot - run: ${{ env.RUN }} "scons -j$(nproc)" - - name: pre-commit - timeout-minutes: 4 - run: ${{ env.RUN }} "unset PYTHONWARNINGS && pre-commit run --all && chmod -R 777 /tmp/pre-commit" + - name: Static analysis + timeout-minutes: 1 + run: ${{ env.RUN }} "scripts/lint/lint.sh" unit_tests: name: unit tests - runs-on: ${{ ((github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }} + runs-on: + - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }} + - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }} steps: - uses: actions/checkout@v4 with: submodules: true - uses: ./.github/workflows/setup-with-retry - with: - docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }} - name: Build openpilot - timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache run: ${{ env.RUN }} "scons -j$(nproc)" - name: Run unit tests - timeout-minutes: 15 + timeout-minutes: ${{ contains(runner.name, 'nsc') && 1 || 20 }} run: | - ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \ - export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && \ - $PYTEST --timeout 60 -m 'not slow' && \ + ${{ env.RUN }} "$PYTEST --collect-only -m 'not slow' &> /dev/null && \ + MAX_EXAMPLES=1 $PYTEST -m 'not slow' && \ ./selfdrive/ui/tests/create_test_translations.sh && \ QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \ - pytest ./selfdrive/ui/tests/test_translations.py" + chmod -R 777 /tmp/comma_download_cache" - name: "Upload coverage to Codecov" uses: codecov/codecov-action@v4 with: @@ -155,27 +153,25 @@ jobs: process_replay: name: process replay - runs-on: ${{ ((github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }} + runs-on: + - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }} + - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }} steps: - uses: actions/checkout@v4 with: submodules: true - uses: ./.github/workflows/setup-with-retry - with: - docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }} - name: Cache test routes id: dependency-cache uses: actions/cache@v4 with: path: .ci_cache/comma_download_cache - key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_regen.py') }} + key: proc-replay-${{ hashFiles('selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_processes.py') }} - name: Build openpilot run: | ${{ env.RUN }} "scons -j$(nproc)" - name: Run replay - timeout-minutes: 30 + timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && 1 || 20 }} run: | ${{ env.RUN }} "coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \ chmod -R 777 /tmp/comma_download_cache && \ @@ -194,15 +190,9 @@ jobs: - name: Upload reference logs if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }} run: | - ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only" - # PYTHONWARNINGS triggers a SyntaxError in onnxruntime - - name: Run model replay with ONNX - timeout-minutes: 4 - run: | - ${{ env.RUN }} "unset PYTHONWARNINGS && \ - ONNXCPU=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \ - coverage combine && coverage xml" + ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python3 selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only" - name: Run regen + if: false timeout-minutes: 4 run: | ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \ @@ -216,33 +206,33 @@ jobs: test_cars: name: cars - runs-on: ${{ ((github.repository == 'commaai/openpilot') && - ((github.event_name != 'pull_request') || - (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }} + runs-on: + - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }} + - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }} strategy: fail-fast: false matrix: - job: [0, 1] + job: [0, 1, 2, 3] steps: - uses: actions/checkout@v4 with: submodules: true - uses: ./.github/workflows/setup-with-retry - name: Cache test routes - id: dependency-cache - uses: ./.github/workflows/auto-cache + id: routes-cache + uses: actions/cache@v4 with: path: .ci_cache/comma_download_cache - key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }} + key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'opendbc/car/tests/routes.py') }}-${{ matrix.job }} - name: Build openpilot run: ${{ env.RUN }} "scons -j$(nproc)" - name: Test car models - timeout-minutes: 20 + timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.routes-cache.outputs.cache-hit == 'true') && 1 || 20 }} run: | - ${{ env.RUN }} "$PYTEST selfdrive/car/tests/test_models.py && \ + ${{ env.RUN }} "MAX_EXAMPLES=1 $PYTEST selfdrive/car/tests/test_models.py && \ chmod -R 777 /tmp/comma_download_cache" env: - NUM_JOBS: 2 + NUM_JOBS: 4 JOB_ID: ${{ matrix.job }} - name: "Upload coverage to Codecov" uses: codecov/codecov-action@v4 @@ -254,7 +244,8 @@ jobs: car_docs_diff: name: PR comments runs-on: ubuntu-latest - if: github.event_name == 'pull_request' + #if: github.event_name == 'pull_request' + if: false # TODO: run this in opendbc? steps: - uses: actions/checkout@v4 with: @@ -264,7 +255,7 @@ jobs: - uses: ./.github/workflows/setup-with-retry - name: Get base car info run: | - ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_docs.py --path /tmp/openpilot_cache/base_car_docs" + ${{ env.RUN }} "scons -j$(nproc) && python3 selfdrive/debug/dump_car_docs.py --path /tmp/openpilot_cache/base_car_docs" sudo chown -R $USER:$USER ${{ github.workspace }} - uses: actions/checkout@v4 with: @@ -276,7 +267,7 @@ jobs: run: | cd current ${{ env.RUN }} "scons -j$(nproc)" - output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_docs") + output=$(${{ env.RUN }} "python3 selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_docs") output="${output//$'\n'/'%0A'}" echo "::set-output name=diff::$output" - name: Find comment @@ -305,24 +296,55 @@ jobs: comment_id: ${{ steps.fc.outputs.comment-id }} }) + simulator_driving: + name: simulator driving + runs-on: + - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }} + - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }} + if: (github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot')) + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: ./.github/workflows/setup-with-retry + - name: Build openpilot + run: | + ${{ env.RUN }} "scons -j$(nproc)" + - name: Driving test + timeout-minutes: 1 + run: | + ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \ + source selfdrive/test/setup_vsound.sh && \ + CI=1 pytest tools/sim/tests/test_metadrive_bridge.py" + create_ui_report: + # This job name needs to be the same as UI_JOB_NAME in ui_preview.yaml name: Create UI Report - runs-on: ubuntu-latest + runs-on: + - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }} + - ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }} steps: - uses: actions/checkout@v4 with: submodules: true - uses: ./.github/workflows/setup-with-retry + - name: caching frames + id: frames-cache + uses: actions/cache@v4 + with: + path: .ci_cache/comma_download_cache + key: ui_screenshots_test_${{ hashFiles('selfdrive/ui/tests/test_ui/run.py') }} - name: Build openpilot run: ${{ env.RUN }} "scons -j$(nproc)" - name: Create Test Report + timeout-minutes: ${{ ((steps.frames-cache.outputs.cache-hit == 'true') && 1 || 3) }} run: > ${{ env.RUN }} "PYTHONWARNINGS=ignore && source selfdrive/test/setup_xvfb.sh && - export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && - python selfdrive/ui/tests/test_ui/run.py" + CACHE_ROOT=/tmp/comma_download_cache python3 selfdrive/ui/tests/test_ui/run.py && + chmod -R 777 /tmp/comma_download_cache" - name: Upload Test Report uses: actions/upload-artifact@v4 with: - name: report - path: selfdrive/ui/tests/test_ui/report + name: report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }} + path: selfdrive/ui/tests/test_ui/report_1/screenshots diff --git a/.github/workflows/setup-pre-commit/action.yaml b/.github/workflows/setup-pre-commit/action.yaml deleted file mode 100644 index f07a106861..0000000000 --- a/.github/workflows/setup-pre-commit/action.yaml +++ /dev/null @@ -1,12 +0,0 @@ -name: 'set up pre-commit environment' - -runs: - using: "composite" - steps: - - uses: ./.github/workflows/auto-cache - with: - path: .ci_cache/pre-commit - key: pre-commit-${{ hashFiles('**/.pre-commit-config.yaml') }} - restore-keys: | - pre-commit- - save: ${{ github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' }} diff --git a/.github/workflows/setup-with-retry/action.yaml b/.github/workflows/setup-with-retry/action.yaml index 369ac45c9a..ad297403cf 100644 --- a/.github/workflows/setup-with-retry/action.yaml +++ b/.github/workflows/setup-with-retry/action.yaml @@ -17,7 +17,6 @@ runs: uses: ./.github/workflows/setup continue-on-error: true with: - docker_hub_pat: ${{ inputs.docker_hub_pat }} is_retried: true - if: steps.setup1.outcome == 'failure' shell: bash @@ -27,7 +26,6 @@ runs: uses: ./.github/workflows/setup continue-on-error: true with: - docker_hub_pat: ${{ inputs.docker_hub_pat }} is_retried: true - if: steps.setup2.outcome == 'failure' shell: bash @@ -36,5 +34,4 @@ runs: if: steps.setup2.outcome == 'failure' uses: ./.github/workflows/setup with: - docker_hub_pat: ${{ inputs.docker_hub_pat }} is_retried: true diff --git a/.github/workflows/setup/action.yaml b/.github/workflows/setup/action.yaml index 970d62030d..818060c3b0 100644 --- a/.github/workflows/setup/action.yaml +++ b/.github/workflows/setup/action.yaml @@ -1,10 +1,6 @@ name: 'openpilot env setup' inputs: - docker_hub_pat: - description: 'Auth token for Docker Hub, required for BuildJet jobs' - required: true - default: '' is_retried: description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly' required: false @@ -20,22 +16,19 @@ runs: echo "You should not run this action directly. Use setup-with-retry instead" exit 1 - # do this after checkout to ensure our custom LFS config is used to pull from GitLab - shell: bash - run: git lfs pull + name: No retries! + run: | + if [ "${{ github.run_attempt }}" -gt 1 ]; then + echo -e "\033[0;31m##################################################" + echo -e "\033[0;31m Retries not allowed! Fix the flaky test! " + echo -e "\033[0;31m##################################################\033[0m" + exit 1 + fi - # on BuildJet runners, must be logged into DockerHub to avoid rate limiting - # https://buildjet.com/for-github-actions/docs/guides/docker + # do this after checkout to ensure our custom LFS config is used to pull from GitLab - shell: bash - if: ${{ contains(runner.name, 'buildjet') && inputs.docker_hub_pat == '' }} - run: | - echo "Need to set the Docker Hub PAT secret as an input to this action" - exit 1 - - name: Login to Docker Hub - if: contains(runner.name, 'buildjet') - shell: bash - run: | - docker login -u adeebshihadeh -p ${{ inputs.docker_hub_pat }} + run: git lfs pull # build cache - id: date @@ -60,4 +53,4 @@ runs: find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \; # build our docker image - shell: bash - run: eval ${{ env.BUILD }} \ No newline at end of file + run: eval ${{ env.BUILD }} diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index 3c86b8e854..d0dead5126 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -5,8 +5,8 @@ on: workflow_dispatch: env: - DAYS_BEFORE_PR_CLOSE: 3 - DAYS_BEFORE_PR_STALE: 14 + DAYS_BEFORE_PR_CLOSE: 2 + DAYS_BEFORE_PR_STALE: 9 jobs: stale: @@ -14,14 +14,14 @@ jobs: steps: - uses: actions/stale@v9 with: - exempt-milestones: true + exempt-all-milestones: true # pull request config stale-pr-message: 'This PR has had no activity for ${{ env.DAYS_BEFORE_PR_STALE }} days. It will be automatically closed in ${{ env.DAYS_BEFORE_PR_CLOSE }} days if there is no activity.' close-pr-message: 'This PR has been automatically closed due to inactivity. Feel free to re-open once activity resumes.' stale-pr-label: stale delete-branch: ${{ github.event.pull_request.head.repo.full_name == 'commaai/openpilot' }} # only delete branches on the main repo - exempt-pr-labels: "ignore stale,needs testing,car port,car" # if wip or it needs testing from the community, don't mark as stale + exempt-pr-labels: "ignore stale,needs testing" # if wip or it needs testing from the community, don't mark as stale days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE }} days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }} diff --git a/.github/workflows/tools_tests.yaml b/.github/workflows/tools_tests.yaml deleted file mode 100644 index 10c5b70910..0000000000 --- a/.github/workflows/tools_tests.yaml +++ /dev/null @@ -1,86 +0,0 @@ -name: tools - -on: - push: - branches: - - master - pull_request: - -concurrency: - group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }} - cancel-in-progress: true - -env: - BASE_IMAGE: openpilot-base - DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} - - BUILD: selfdrive/test/docker_build.sh base - - RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c - - -jobs: - simulator_build: - name: simulator docker build - runs-on: ubuntu-latest - if: github.repository == 'commaai/openpilot' - timeout-minutes: 45 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: ./.github/workflows/setup-with-retry - - name: Setup to push to repo - if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot' - run: | - echo "PUSH_IMAGE=true" >> "$GITHUB_ENV" - $DOCKER_LOGIN - - name: Build and push sim image - run: | - selfdrive/test/docker_build.sh sim - - simulator_driving: - name: simulator driving - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: ./.github/workflows/setup-with-retry - - name: Build openpilot - run: | - ${{ env.RUN }} "scons -j$(nproc)" - - name: Run bridge test - run: | - ${{ env.RUN }} "export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && \ - source selfdrive/test/setup_xvfb.sh && \ - source selfdrive/test/setup_vsound.sh && \ - CI=1 pytest tools/sim/tests/test_metadrive_bridge.py" - - devcontainer: - name: devcontainer - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: ./.github/workflows/setup-with-retry - - name: Use local image for testing devcontainer with latest base image - run: | - echo "USE_LOCAL_IMAGE=true" >> "$GITHUB_ENV" - - name: Setup Dev Container CLI - run: npm install -g @devcontainers/cli - - name: Build dev container image - run: ./scripts/retry.sh devcontainer build --workspace-folder . - - name: Run dev container - run: | - mkdir -p /tmp/devcontainer_scons_cache/ - cp -r $GITHUB_WORKSPACE/.ci_cache/scons_cache/. /tmp/devcontainer_scons_cache/ - devcontainer up --workspace-folder . - - name: Test environment - run: | - devcontainer exec --workspace-folder . scons -j$(nproc) cereal/ common/ - devcontainer exec --workspace-folder . pip install pip-install-test - devcontainer exec --workspace-folder . touch /home/batman/.comma/auth.json - devcontainer exec --workspace-folder . sudo touch /root/test.txt diff --git a/.github/workflows/ui_preview.yaml b/.github/workflows/ui_preview.yaml new file mode 100644 index 0000000000..7936192ca8 --- /dev/null +++ b/.github/workflows/ui_preview.yaml @@ -0,0 +1,161 @@ +name: "ui preview" +on: + push: + branches: + - master + pull_request_target: + types: [assigned, opened, synchronize, reopened, edited] + branches: + - 'master' + paths: + - 'selfdrive/ui/**' + workflow_dispatch: + +env: + UI_JOB_NAME: "Create UI Report" + REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }} + SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }} + BRANCH_NAME: "openpilot/pr-${{ github.event.number }}" + +jobs: + preview: + if: github.repository == 'commaai/openpilot' + name: preview + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + pull-requests: write + actions: read + steps: + - name: Waiting for ui generation to start + run: sleep 30 + + - name: Waiting for ui generation to end + uses: lewagon/wait-on-check-action@v1.3.4 + with: + ref: ${{ env.SHA }} + check-name: ${{ env.UI_JOB_NAME }} + repo-token: ${{ secrets.GITHUB_TOKEN }} + allowed-conclusions: success + wait-interval: 20 + + - name: Getting workflow run ID + id: get_run_id + run: | + echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?[0-9]+)") | .number')" >> $GITHUB_OUTPUT + + - name: Getting proposed ui + id: download-artifact + uses: dawidd6/action-download-artifact@v6 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + run_id: ${{ steps.get_run_id.outputs.run_id }} + search_artifacts: true + name: report-1-${{ env.REPORT_NAME }} + path: ${{ github.workspace }}/pr_ui + + - name: Getting master ui + uses: actions/checkout@v4 + with: + repository: commaai/ci-artifacts + ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }} + path: ${{ github.workspace }}/master_ui + ref: openpilot_master_ui + + - name: Saving new master ui + if: github.ref == 'refs/heads/master' && github.event_name == 'push' + working-directory: ${{ github.workspace }}/master_ui + run: | + git checkout --orphan=new_master_ui + git rm -rf * + git branch -D openpilot_master_ui + git branch -m openpilot_master_ui + git config user.name "GitHub Actions Bot" + git config user.email "<>" + mv ${{ github.workspace }}/pr_ui/*.png . + git add . + git commit -m "screenshots for commit ${{ env.SHA }}" + git push origin openpilot_master_ui --force + + - name: Finding diff + if: github.event_name == 'pull_request_target' + id: find_diff + run: >- + sudo apt-get install -y imagemagick + + scenes="homescreen settings_device settings_software settings_toggles settings_developer offroad_alert update_available prime onroad onroad_disengaged onroad_override onroad_sidebar onroad_wide onroad_wide_sidebar onroad_alert_small onroad_alert_mid onroad_alert_full driver_camera body keyboard keyboard_uppercase" + A=($scenes) + + DIFF="" + TABLE="
All Screenshots" + TABLE="${TABLE}" + + for ((i=0; i<${#A[*]}; i=i+1)); + do + + if ! compare -fuzz 2% -highlight-color DeepSkyBlue1 -lowlight-color Black -compose Src ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png; then + convert ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png -transparent black mask.png + composite mask.png ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png + convert -delay 100 ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png -loop 0 ${{ github.workspace }}/pr_ui/${A[$i]}_diff.gif + + mv ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_master_ref.png + + DIFF="${DIFF}
" + DIFF="${DIFF}${A[$i]} : \$\${\\color{red}\\text{DIFFERENT}}\$\$" + DIFF="${DIFF}
" + + DIFF="${DIFF}" + DIFF="${DIFF} " + DIFF="${DIFF} " + DIFF="${DIFF}" + + DIFF="${DIFF}" + DIFF="${DIFF} " + DIFF="${DIFF} " + DIFF="${DIFF}" + + DIFF="${DIFF}
master proposed
diff composite diff
" + DIFF="${DIFF}
" + else + rm -f ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png + fi + + INDEX=$(($i % 2)) + if [[ $INDEX -eq 0 ]]; then + TABLE="${TABLE}" + fi + TABLE="${TABLE} " + if [[ $INDEX -eq 1 || $(($i + 1)) -eq ${#A[*]} ]]; then + TABLE="${TABLE}" + fi + done + + TABLE="${TABLE}" + + echo "DIFF=$DIFF$TABLE" >> "$GITHUB_OUTPUT" + + - name: Saving proposed ui + if: github.event_name == 'pull_request_target' + working-directory: ${{ github.workspace }}/master_ui + run: | + git config user.name "GitHub Actions Bot" + git config user.email "<>" + git checkout --orphan=${{ env.BRANCH_NAME }} + git rm -rf * + mv ${{ github.workspace }}/pr_ui/* . + git add . + git commit -m "screenshots for PR #${{ github.event.number }}" + git push origin ${{ env.BRANCH_NAME }} --force + + - name: Comment Screenshots on PR + if: github.event_name == 'pull_request_target' + uses: thollander/actions-comment-pull-request@v2 + with: + message: | + + ## UI Preview + ${{ steps.find_diff.outputs.DIFF }} + comment_tag: run_id_screenshots + pr_number: ${{ github.event.number }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 370335e6a0..4562e47817 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ model2.png a.out .hypothesis +/docs_site/ + *.dylib *.DSYM *.d @@ -43,6 +45,7 @@ persist selfdrive/pandad/pandad cereal/services.h cereal/gen +cereal/messaging/bridge selfdrive/logcatd/logcatd selfdrive/mapd/default_speeds_by_region.json system/proclogd/proclogd @@ -52,9 +55,6 @@ selfdrive/test/longitudinal_maneuvers/out selfdrive/car/tests/cars_dump system/camerad/camerad system/camerad/test/ae_gray_test -selfdrive/modeld/_modeld -selfdrive/modeld/_dmonitoringmodeld -/src/ notebooks hyperthneed @@ -76,6 +76,7 @@ selfdrive/modeld/models/*.thneed selfdrive/modeld/models/*.pkl *.bz2 +*.zst build/ @@ -83,3 +84,22 @@ build/ poetry.toml Pipfile + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide diff --git a/.gitmodules b/.gitmodules index 4952fcbeae..54c7393986 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,20 +2,17 @@ path = panda url = ../../commaai/panda.git [submodule "opendbc"] - path = opendbc + path = opendbc_repo url = ../../commaai/opendbc.git [submodule "msgq"] - path = msgq + path = msgq_repo url = ../../commaai/msgq.git [submodule "rednose_repo"] path = rednose_repo url = ../../commaai/rednose.git -[submodule "body"] - path = body - url = ../../commaai/body.git [submodule "teleoprtc_repo"] path = teleoprtc_repo url = ../../commaai/teleoprtc [submodule "tinygrad"] path = tinygrad_repo - url = https://github.com/geohot/tinygrad.git + url = https://github.com/commaai/tinygrad.git diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 75ddcfb616..0000000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,103 +0,0 @@ -exclude: '^(tinygrad_repo)' -repos: -- repo: meta - hooks: - - id: check-hooks-apply - - id: check-useless-excludes -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 - hooks: - - id: check-ast - exclude: '^(third_party)/' - - id: check-json - exclude: '.devcontainer/devcontainer.json|.vscode/' # these support JSON with comments - - id: check-toml - - id: check-xml - - id: check-yaml - - id: check-merge-conflict - - id: check-symlinks - - id: check-executables-have-shebangs - - id: check-shebang-scripts-are-executable - - id: check-added-large-files - exclude: '(docs/CARS.md)|(poetry.lock)|(third_party/acados/include/blasfeo/include/blasfeo_d_kernel.h)' - args: - - --maxkb=120 - - --enforce-all -- repo: https://github.com/codespell-project/codespell - rev: v2.2.6 - hooks: - - id: codespell - exclude: '^(third_party/)|(body/)|(msgq/)|(panda/)|(opendbc/)|(rednose/)|(rednose_repo/)|(teleoprtc/)|(teleoprtc_repo/)|(selfdrive/ui/translations/.*.ts)|(poetry.lock)' - args: - # if you've got a short variable name that's getting flagged, add it here - - -L bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,preints - - --builtins clear,rare,informal,usage,code,names,en-GB_to_en-US -- repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.4 - hooks: - - id: ruff - exclude: '^(third_party/)|(msgq/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)' -- repo: local - hooks: - - id: mypy - name: mypy - entry: mypy - language: system - types: [python] - args: - - --local-partial-types - - --explicit-package-bases - exclude: '^(third_party/)|(body/)|(msgq/)|(opendbc/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)' -- repo: local - hooks: - - id: cppcheck - name: cppcheck - entry: cppcheck - language: system - types: [c++] - exclude: '^(third_party/)|(msgq/)|(body/)|(rednose/)|(rednose_repo/)|(opendbc/)|(panda/)|(tools/)|(selfdrive/modeld/thneed/debug/)|(selfdrive/modeld/test/)|(selfdrive/camerad/test/)|(installer/)' - args: - - --error-exitcode=1 - - --language=c++ - - --quiet - - --force - - -j8 -- repo: https://github.com/cpplint/cpplint - rev: 1.6.1 - hooks: - - id: cpplint - exclude: '^(third_party/)|(msgq/)|(body/)|(rednose/)|(rednose_repo/)|(opendbc/)|(panda/)|(generated/)' - args: - - --quiet - - --counting=total - - --linelength=240 - # https://google.github.io/styleguide/cppguide.html - # relevant rules are whitelisted, see all options with: cpplint --filter= - - --filter=-build,-legal,-readability,-runtime,-whitespace,+build/include_subdir,+build/forward_decl,+build/include_what_you_use,+build/deprecated,+whitespace/comma,+whitespace/line_length,+whitespace/empty_if_body,+whitespace/empty_loop_body,+whitespace/empty_conditional_body,+whitespace/forcolon,+whitespace/parens,+whitespace/semicolon,+whitespace/tab,+readability/braces -- repo: https://github.com/MarcoGorelli/cython-lint - rev: v0.16.2 - hooks: - - id: cython-lint - exclude: '^(third_party/)|(msgq/)|(body/)|(rednose/)|(rednose_repo/)|(opendbc/)|(panda/)|(generated/)' - args: - - --max-line-length=240 - - --ignore=E111, E302, E305 -- repo: local - hooks: - - id: test_translations - name: test translations - entry: pytest selfdrive/ui/tests/test_translations.py - language: system - pass_filenames: false - files: '^selfdrive/ui/translations/' -- repo: https://github.com/python-poetry/poetry - rev: '1.8.0' - hooks: - - id: poetry-check - name: validate poetry lock - args: - - --lock -- repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.28.3 - hooks: - - id: check-github-workflows diff --git a/.python-version b/.python-version deleted file mode 100644 index 0c7d5f5f5d..0000000000 --- a/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.11.4 diff --git a/.vscode/launch.json b/.vscode/launch.json index 3b3953c3f4..a6b341d9ea 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -7,7 +7,6 @@ "description": "Select the process to debug", "options": [ "selfdrive/controls/controlsd.py", - "selfdrive/navd/navd.py", "system/timed/timed.py", "tools/sim/run_bridge.py" ] diff --git a/Dockerfile.openpilot b/Dockerfile.openpilot index 9aa1aa8b74..f34caba81b 100644 --- a/Dockerfile.openpilot +++ b/Dockerfile.openpilot @@ -1,29 +1,13 @@ FROM ghcr.io/commaai/openpilot-base:latest -ENV PYTHONUNBUFFERED 1 +ENV PYTHONUNBUFFERED=1 -ENV OPENPILOT_PATH /home/batman/openpilot -ENV PYTHONPATH ${OPENPILOT_PATH}:${PYTHONPATH} +ENV OPENPILOT_PATH=/home/batman/openpilot +ENV PYTHONPATH=${OPENPILOT_PATH}:${PYTHONPATH} RUN mkdir -p ${OPENPILOT_PATH} WORKDIR ${OPENPILOT_PATH} -COPY SConstruct ${OPENPILOT_PATH} - -COPY ./openpilot ${OPENPILOT_PATH}/openpilot -COPY ./third_party ${OPENPILOT_PATH}/third_party -COPY ./site_scons ${OPENPILOT_PATH}/site_scons -COPY ./rednose ${OPENPILOT_PATH}/rednose -COPY ./rednose_repo/site_scons ${OPENPILOT_PATH}/rednose_repo/site_scons -COPY ./tools ${OPENPILOT_PATH}/tools -COPY ./release ${OPENPILOT_PATH}/release -COPY ./common ${OPENPILOT_PATH}/common -COPY ./opendbc ${OPENPILOT_PATH}/opendbc -COPY ./cereal ${OPENPILOT_PATH}/cereal -COPY ./msgq ${OPENPILOT_PATH}/msgq -COPY ./panda ${OPENPILOT_PATH}/panda -COPY ./selfdrive ${OPENPILOT_PATH}/selfdrive -COPY ./system ${OPENPILOT_PATH}/system -COPY ./body ${OPENPILOT_PATH}/body +COPY . ${OPENPILOT_PATH}/ RUN scons --cache-readonly -j$(nproc) diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base index 0789a39c3e..44d8d95e95 100644 --- a/Dockerfile.openpilot_base +++ b/Dockerfile.openpilot_base @@ -1,21 +1,21 @@ -FROM ubuntu:20.04 +FROM ubuntu:24.04 -ENV PYTHONUNBUFFERED 1 +ENV PYTHONUNBUFFERED=1 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ - apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot && \ + apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot python3-tk python3-dev && \ rm -rf /var/lib/apt/lists/* RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 COPY tools/install_ubuntu_dependencies.sh /tmp/tools/ -RUN INSTALL_EXTRA_PACKAGES=no /tmp/tools/install_ubuntu_dependencies.sh && \ +RUN /tmp/tools/install_ubuntu_dependencies.sh && \ rm -rf /var/lib/apt/lists/* /tmp/* && \ - cd /usr/lib/gcc/arm-none-eabi/9.2.1 && \ + cd /usr/lib/gcc/arm-none-eabi/* && \ rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp # Add OpenCL @@ -30,53 +30,52 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gcc-arm-none-eabi \ tmux \ vim \ - lsb-core \ libx11-6 \ + wget \ && rm -rf /var/lib/apt/lists/* -ARG INTEL_DRIVER=l_opencl_p_18.1.0.015.tgz -ARG INTEL_DRIVER_URL=https://registrationcenter-download.intel.com/akdlm/irc_nas/vcp/15532 -RUN mkdir -p /tmp/opencl-driver-intel - -RUN cd /tmp/opencl-driver-intel && \ - echo INTEL_DRIVER is $INTEL_DRIVER && \ - curl -O $INTEL_DRIVER_URL/$INTEL_DRIVER && \ - tar -xzf $INTEL_DRIVER && \ - for i in $(basename $INTEL_DRIVER .tgz)/rpm/*.rpm; do alien --to-deb $i; done && \ - dpkg -i *.deb && \ - rm -rf $INTEL_DRIVER $(basename $INTEL_DRIVER .tgz) *.deb && \ +RUN mkdir -p /tmp/opencl-driver-intel && \ + cd /tmp/opencl-driver-intel && \ + wget https://github.com/intel/llvm/releases/download/2024-WW14/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \ + wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.12.0/oneapi-tbb-2021.12.0-lin.tgz && \ + mkdir -p /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \ + cd /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \ + tar -zxvf /tmp/opencl-driver-intel/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \ mkdir -p /etc/OpenCL/vendors && \ - echo /opt/intel/opencl_compilers_and_libraries_18.1.0.015/linux/compiler/lib/intel64_lin/libintelocl.so > /etc/OpenCL/vendors/intel.icd && \ + echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64/libintelocl.so > /etc/OpenCL/vendors/intel_expcpu.icd && \ + cd /opt/intel && \ + tar -zxvf /tmp/opencl-driver-intel/oneapi-tbb-2021.12.0-lin.tgz && \ + ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \ + ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \ + ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so.12 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \ + ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so.2 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \ + mkdir -p /etc/ld.so.conf.d && \ + echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf && \ + ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf && \ cd / && \ rm -rf /tmp/opencl-driver-intel -ENV NVIDIA_VISIBLE_DEVICES all -ENV NVIDIA_DRIVER_CAPABILITIES graphics,utility,compute -ENV QTWEBENGINE_DISABLE_SANDBOX 1 +ENV NVIDIA_VISIBLE_DEVICES=all +ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute +ENV QTWEBENGINE_DISABLE_SANDBOX=1 RUN dbus-uuidgen > /etc/machine-id ARG USER=batman -ARG USER_UID=1000 +ARG USER_UID=1001 RUN useradd -m -s /bin/bash -u $USER_UID $USER RUN usermod -aG sudo $USER RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER $USER -ENV POETRY_VIRTUALENVS_CREATE=false -ENV PYENV_VERSION=3.11.4 -ENV PYENV_ROOT="/home/$USER/pyenv" -ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH" - -COPY --chown=$USER pyproject.toml poetry.lock .python-version /tmp/ -COPY --chown=$USER tools/install_python_dependencies.sh /tmp/tools/ +COPY --chown=$USER pyproject.toml uv.lock /home/$USER +COPY --chown=$USER tools/install_python_dependencies.sh /home/$USER/tools/ -RUN cd /tmp && \ +ENV VIRTUAL_ENV=/home/$USER/.venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +RUN cd /home/$USER && \ tools/install_python_dependencies.sh && \ - rm -rf /tmp/* && \ - rm -rf /home/$USER/.cache && \ - find /home/$USER/pyenv -type d -name ".git" | xargs rm -rf && \ - rm -rf /home/$USER/pyenv/versions/3.11.4/lib/python3.11/test + rm -rf tools/ pyproject.toml uv.lock .cache USER root RUN sudo git config --global --add safe.directory /tmp/openpilot diff --git a/Jenkinsfile b/Jenkinsfile index 321237716b..656f1055eb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -12,10 +12,12 @@ def retryWithDelay(int maxRetries, int delay, Closure body) { def device(String ip, String step_label, String cmd) { withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) { def ssh_cmd = """ -ssh -tt -o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=2 -o BatchMode=yes -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' /usr/bin/bash <<'END' +ssh -o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=2 -o BatchMode=yes -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' exec /usr/bin/bash <<'END' set -e +export TERM=xterm-256color + shopt -s huponexit # kill all child processes when the shell exits export CI=1 @@ -25,8 +27,9 @@ export TEST_DIR=${env.TEST_DIR} export SOURCE_DIR=${env.SOURCE_DIR} export GIT_BRANCH=${env.GIT_BRANCH} export GIT_COMMIT=${env.GIT_COMMIT} +export CI_ARTIFACTS_TOKEN=${env.CI_ARTIFACTS_TOKEN} +export GITHUB_COMMENTS_TOKEN=${env.GITHUB_COMMENTS_TOKEN} export AZURE_TOKEN='${env.AZURE_TOKEN}' -export MAPBOX_TOKEN='${env.MAPBOX_TOKEN}' # only use 1 thread for tici tests since most require HIL export PYTEST_ADDOPTS="-n 0" @@ -63,9 +66,7 @@ fi ln -snf ${env.TEST_DIR} /data/pythonpath cd ${env.TEST_DIR} || true -${cmd} -exit 0 - +time ${cmd} END""" sh script: ssh_cmd, label: step_label @@ -78,17 +79,38 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps) return } + if (isReplay()) { + error("REPLAYING TESTS IS NOT ALLOWED. FIX THEM INSTEAD.") + } + def extra = extra_env.collect { "export ${it}" }.join('\n'); def branch = env.BRANCH_NAME ?: 'master'; + def gitDiff = sh returnStdout: true, script: 'curl -s -H "Authorization: Bearer ${GITHUB_COMMENTS_TOKEN}" https://api.github.com/repos/commaai/openpilot/compare/master...${GIT_BRANCH} | jq .files[].filename || echo "/"', label: 'Getting changes' lock(resource: "", label: deviceType, inversePrecedence: true, variable: 'device_ip', quantity: 1, resourceSelectStrategy: 'random') { docker.image('ghcr.io/commaai/alpine-ssh').inside('--user=root') { - timeout(time: 20, unit: 'MINUTES') { + timeout(time: 35, unit: 'MINUTES') { retry (3) { + def date = sh(script: 'date', returnStdout: true).trim(); + device(device_ip, "set time", "date -s '" + date + "'") device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh")) } steps.each { item -> - device(device_ip, item[0], item[1]) + def name = item[0] + def cmd = item[1] + + def args = item[2] + def diffPaths = args.diffPaths ?: [] + def cmdTimeout = args.timeout ?: 9999 + + if (branch != "master" && diffPaths && !hasPathChanged(gitDiff, diffPaths)) { + println "Skipping ${name}: no changes in ${diffPaths}." + return + } else { + timeout(time: cmdTimeout, unit: 'SECONDS') { + device(device_ip, name, cmd) + } + } } } } @@ -96,52 +118,43 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps) } } -def pcStage(String stageName, Closure body) { - node { - stage(stageName) { - if (currentBuild.result != null) { - return - } - - checkout scm - - def dockerArgs = "--user=batman -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/scons_cache:/tmp/scons_cache -e PYTHONPATH=${env.WORKSPACE} --cpus=8 --memory 16g -e PYTEST_ADDOPTS='-n8'"; - - def openpilot_base = retryWithDelay (3, 15) { - return docker.build("openpilot-base:build-${env.GIT_COMMIT}", "-f Dockerfile.openpilot_base .") - } - - lock(resource: "", label: 'pc', inversePrecedence: true, quantity: 1) { - openpilot_base.inside(dockerArgs) { - timeout(time: 20, unit: 'MINUTES') { - try { - retryWithDelay (3, 15) { - sh "git config --global --add safe.directory '*'" - sh "git submodule update --init --recursive" - sh "git lfs pull" - } - body() - } finally { - sh "rm -rf ${env.WORKSPACE}/* || true" - sh "rm -rf .* || true" - } - } - } - } +def hasPathChanged(String gitDiff, List paths) { + for (path in paths) { + if (gitDiff.contains(path)) { + return true } } + return false +} + +def isReplay() { + def replayClass = "org.jenkinsci.plugins.workflow.cps.replay.ReplayCause" + return currentBuild.rawBuild.getCauses().any{ cause -> cause.toString().contains(replayClass) } } def setupCredentials() { withCredentials([ string(credentialsId: 'azure_token', variable: 'AZURE_TOKEN'), - string(credentialsId: 'mapbox_token', variable: 'MAPBOX_TOKEN') ]) { env.AZURE_TOKEN = "${AZURE_TOKEN}" - env.MAPBOX_TOKEN = "${MAPBOX_TOKEN}" + } + + withCredentials([ + string(credentialsId: 'ci_artifacts_pat', variable: 'CI_ARTIFACTS_TOKEN'), + ]) { + env.CI_ARTIFACTS_TOKEN = "${CI_ARTIFACTS_TOKEN}" + } + + withCredentials([ + string(credentialsId: 'post_comments_github_pat', variable: 'GITHUB_COMMENTS_TOKEN'), + ]) { + env.GITHUB_COMMENTS_TOKEN = "${GITHUB_COMMENTS_TOKEN}" } } +def step(String name, String cmd, Map args = [:]) { + return [name, cmd, args] +} node { env.CI = "1" @@ -166,82 +179,90 @@ node { try { if (env.BRANCH_NAME == 'devel-staging') { deviceStage("build release3-staging", "tici-needs-can", [], [ - ["build release3-staging", "RELEASE_BRANCH=release3-staging $SOURCE_DIR/release/build_release.sh"], + step("build release3-staging", "RELEASE_BRANCH=release3-staging $SOURCE_DIR/release/build_release.sh"), ]) } if (env.BRANCH_NAME == 'master-ci') { - deviceStage("build nightly", "tici-needs-can", [], [ - ["build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"], - ]) + parallel ( + 'nightly': { + deviceStage("build nightly", "tici-needs-can", [], [ + step("build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"), + ]) + }, + 'nightly-dev': { + deviceStage("build nightly-dev", "tici-needs-can", [], [ + step("build nightly-dev", "PANDA_DEBUG_BUILD=1 RELEASE_BRANCH=nightly-dev $SOURCE_DIR/release/build_release.sh"), + ]) + }, + ) } if (!env.BRANCH_NAME.matches(excludeRegex)) { parallel ( // tici tests 'onroad tests': { - deviceStage("onroad", "tici-needs-can", [], [ + deviceStage("onroad", "tici-needs-can", ["UNSAFE=1"], [ // TODO: ideally, this test runs in master-ci, but it takes 5+m to build it //["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR $SOURCE_DIR/scripts/retry.sh ./build_devel.sh"], - ["build openpilot", "cd system/manager && ./build.py"], - ["check dirty", "release/check-dirty.sh"], - ["onroad tests", "pytest selfdrive/test/test_onroad.py -s"], - ["time to onroad", "pytest selfdrive/test/test_time_to_onroad.py"], + step("build openpilot", "cd system/manager && ./build.py"), + step("check dirty", "release/check-dirty.sh"), + step("onroad tests", "pytest selfdrive/test/test_onroad.py -s", [timeout: 60]), + //["time to onroad", "pytest selfdrive/test/test_time_to_onroad.py"], ]) }, 'HW + Unit Tests': { deviceStage("tici-hardware", "tici-common", ["UNSAFE=1"], [ - ["build", "cd system/manager && ./build.py"], - ["test pandad", "pytest selfdrive/pandad/tests/test_pandad.py"], - ["test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"], - ["test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py"], - ["test pigeond", "pytest system/ubloxd/tests/test_pigeond.py"], - ["test manager", "pytest system/manager/test/test_manager.py"], + step("build", "cd system/manager && ./build.py"), + step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]), + step("test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"), + step("test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py", [diffPaths: ["system/loggerd/"]]), + step("test pigeond", "pytest system/ubloxd/tests/test_pigeond.py", [diffPaths: ["system/ubloxd/"]]), + step("test manager", "pytest system/manager/test/test_manager.py"), ]) }, 'loopback': { deviceStage("loopback", "tici-loopback", ["UNSAFE=1"], [ - ["build openpilot", "cd system/manager && ./build.py"], - ["test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"], + step("build openpilot", "cd system/manager && ./build.py"), + step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"), ]) }, 'camerad': { deviceStage("AR0231", "tici-ar0231", ["UNSAFE=1"], [ - ["build", "cd system/manager && ./build.py"], - ["test camerad", "pytest system/camerad/test/test_camerad.py"], - ["test exposure", "pytest system/camerad/test/test_exposure.py"], + step("build", "cd system/manager && ./build.py"), + step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]), + step("test exposure", "pytest system/camerad/test/test_exposure.py"), ]) deviceStage("OX03C10", "tici-ox03c10", ["UNSAFE=1"], [ - ["build", "cd system/manager && ./build.py"], - ["test camerad", "pytest system/camerad/test/test_camerad.py"], - ["test exposure", "pytest system/camerad/test/test_exposure.py"], + step("build", "cd system/manager && ./build.py"), + step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]), + step("test exposure", "pytest system/camerad/test/test_exposure.py"), ]) }, 'sensord': { deviceStage("LSM + MMC", "tici-lsmc", ["UNSAFE=1"], [ - ["build", "cd system/manager && ./build.py"], - ["test sensord", "pytest system/sensord/tests/test_sensord.py"], + step("build", "cd system/manager && ./build.py"), + step("test sensord", "pytest system/sensord/tests/test_sensord.py"), ]) deviceStage("BMX + LSM", "tici-bmx-lsm", ["UNSAFE=1"], [ - ["build", "cd system/manager && ./build.py"], - ["test sensord", "pytest system/sensord/tests/test_sensord.py"], + step("build", "cd system/manager && ./build.py"), + step("test sensord", "pytest system/sensord/tests/test_sensord.py"), ]) }, 'replay': { deviceStage("model-replay", "tici-replay", ["UNSAFE=1"], [ - ["build", "cd system/manager && ./build.py"], - ["model replay", "selfdrive/test/process_replay/model_replay.py"], + step("build", "cd system/manager && ./build.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]), + step("model replay", "selfdrive/test/process_replay/model_replay.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]), ]) }, 'tizi': { deviceStage("tizi", "tizi", ["UNSAFE=1"], [ - ["build openpilot", "cd system/manager && ./build.py"], - ["test pandad loopback", "SINGLE_PANDA=1 pytest selfdrive/pandad/tests/test_pandad_loopback.py"], - ["test pandad spi", "pytest selfdrive/pandad/tests/test_pandad_spi.py"], - ["test pandad", "pytest selfdrive/pandad/tests/test_pandad.py"], - ["test amp", "pytest system/hardware/tici/tests/test_amplifier.py"], - ["test hw", "pytest system/hardware/tici/tests/test_hardware.py"], - ["test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py"], + step("build openpilot", "cd system/manager && ./build.py"), + step("test pandad loopback", "SINGLE_PANDA=1 pytest selfdrive/pandad/tests/test_pandad_loopback.py"), + step("test pandad spi", "pytest selfdrive/pandad/tests/test_pandad_spi.py"), + step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]), + step("test amp", "pytest system/hardware/tici/tests/test_amplifier.py"), + step("test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py", [diffPaths: ["system/qcomgpsd/"]]), ]) }, diff --git a/README.md b/README.md index db74ee6a1a..77002481d3 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,34 @@ -[![openpilot on the comma 3X](https://github.com/commaai/openpilot/assets/8762862/f09e6d29-db2d-4179-80c2-51e8d92bdb5c)](https://comma.ai/shop/comma-3x) +
+ +

openpilot

+ +

+ openpilot is an operating system for robotics. +
+ Currently, it upgrades the driver assistance system in 275+ supported cars. +

+ +

+ Docs + · + Roadmap + · + Contribute + · + Community + · + Try it on a comma 3X +

+ +Quick start: `bash <(curl -fsSL openpilot.comma.ai)` -What is openpilot? ------- +![openpilot tests](https://github.com/commaai/openpilot/actions/workflows/selfdrive_tests.yaml/badge.svg) +[![codecov](https://codecov.io/gh/commaai/openpilot/branch/master/graph/badge.svg)](https://codecov.io/gh/commaai/openpilot) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) +[![X Follow](https://img.shields.io/twitter/follow/comma_ai)](https://x.com/comma_ai) +[![Discord](https://img.shields.io/discord/469524606043160576)](https://discord.comma.ai) -[openpilot](http://github.com/commaai/openpilot) is an open source driver assistance system. Currently, openpilot performs the functions of Adaptive Cruise Control (ACC), Automated Lane Centering (ALC), Forward Collision Warning (FCW), and Lane Departure Warning (LDW) for a growing variety of [supported car makes, models, and model years](docs/CARS.md). In addition, while openpilot is engaged, a camera-based Driver Monitoring (DM) feature alerts distracted and asleep drivers. See more about [the vehicle integration](docs/INTEGRATION.md) and [limitations](docs/LIMITATIONS.md). +
@@ -13,17 +38,26 @@ What is openpilot?
-To start using openpilot in a car + +Using openpilot in a car ------ To use openpilot in a car, you need four things: 1. **Supported Device:** a comma 3/3X, available at [comma.ai/shop](https://comma.ai/shop/comma-3x). 2. **Software:** The setup procedure for the comma 3/3X allows users to enter a URL for custom software. Use the URL `openpilot.comma.ai` to install the release version. -3. **Supported Car:** Ensure that you have one of [the 250+ supported cars](docs/CARS.md). +3. **Supported Car:** Ensure that you have one of [the 275+ supported cars](docs/CARS.md). 4. **Car Harness:** You will also need a [car harness](https://comma.ai/shop/car-harness) to connect your comma 3/3X to your car. We have detailed instructions for [how to install the harness and device in a car](https://comma.ai/setup). Note that it's possible to run openpilot on [other hardware](https://blog.comma.ai/self-driving-car-for-free/), although it's not plug-and-play. +### Branches +| branch | URL | description | +|------------------|----------------------------------------|-------------------------------------------------------------------------------------| +| `release3` | openpilot.comma.ai | This is openpilot's release branch. | +| `release3-staging` | openpilot-test.comma.ai | This is the staging branch for releases. Use it to get new releases slightly early. | +| `nightly` | openpilot-nightly.comma.ai | This is the bleeding edge development branch. Do not expect this to be stable. | +| `nightly-dev` | installer.comma.ai/commaai/nightly-dev | Same as nightly, but includes experimental development features for some cars. | + To start developing openpilot ------ @@ -36,7 +70,7 @@ openpilot is developed by [comma](https://comma.ai/) and by users like you. We w * Code documentation lives at https://docs.comma.ai * Information about running openpilot lives on the [community wiki](https://github.com/commaai/openpilot/wiki) -Want to get paid to work on openpilot? [comma is hiring](https://comma.ai/jobs#open-positions) and offers lots of [bounties](docs/BOUNTIES.md) for external contributors. +Want to get paid to work on openpilot? [comma is hiring](https://comma.ai/jobs#open-positions) and offers lots of [bounties](https://comma.ai/bounties) for external contributors. Safety and Testing ---- @@ -49,18 +83,6 @@ Safety and Testing * panda has additional hardware-in-the-loop [tests](https://github.com/commaai/panda/blob/master/Jenkinsfile). * We run the latest openpilot in a testing closet containing 10 comma devices continuously replaying routes. -User Data and comma Account ------- - -By default, openpilot uploads the driving data to our servers. You can also access your data through [comma connect](https://connect.comma.ai/). We use your data to train better models and improve openpilot for everyone. - -openpilot is open source software: the user is free to disable data collection if they wish to do so. - -openpilot logs the road-facing cameras, CAN, GPS, IMU, magnetometer, thermal sensors, crashes, and operating system logs. -The driver-facing camera is only logged if you explicitly opt-in in settings. The microphone is not recorded. - -By using openpilot, you agree to [our Privacy Policy](https://comma.ai/privacy). You understand that use of this software or its related services will generate certain types of user data, which may be logged and stored at the sole discretion of comma. By accepting this agreement, you grant an irrevocable, perpetual, worldwide right to comma for the use of this data. - Licensing ------ @@ -72,9 +94,14 @@ Any user of this software shall indemnify and hold harmless Comma.ai, Inc. and i YOU ARE RESPONSIBLE FOR COMPLYING WITH LOCAL LAWS AND REGULATIONS. NO WARRANTY EXPRESSED OR IMPLIED.** ---- +User Data and comma Account +------ - +By default, openpilot uploads the driving data to our servers. You can also access your data through [comma connect](https://connect.comma.ai/). We use your data to train better models and improve openpilot for everyone. -![openpilot tests](https://github.com/commaai/openpilot/actions/workflows/selfdrive_tests.yaml/badge.svg) -[![codecov](https://codecov.io/gh/commaai/openpilot/branch/master/graph/badge.svg)](https://codecov.io/gh/commaai/openpilot) +openpilot is open source software: the user is free to disable data collection if they wish to do so. + +openpilot logs the road-facing cameras, CAN, GPS, IMU, magnetometer, thermal sensors, crashes, and operating system logs. +The driver-facing camera is only logged if you explicitly opt-in in settings. The microphone is not recorded. + +By using openpilot, you agree to [our Privacy Policy](https://comma.ai/privacy). You understand that use of this software or its related services will generate certain types of user data, which may be logged and stored at the sole discretion of comma. By accepting this agreement, you grant an irrevocable, perpetual, worldwide right to comma for the use of this data. diff --git a/RELEASES.md b/RELEASES.md index 31fdd1561b..d326af1d0a 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,13 +1,36 @@ -Version 0.9.7 (2024-06-11) +Version 0.9.9 (2025-03-30) +======================== +* Coming soon + * Rivian support + * F-150 & Mach-E support + * Tesla Model 3 support + +Version 0.9.8 (2025-01-30) +======================== +* New driving monitoring model + * Reduced false positives related to passengers +* Image processing pipeline moved to the ISP + * More GPU time for driving models + * Power draw reduced 0.5W, which means your device runs cooler +* Added toggle to enable driver monitoring even when openpilot is not engaged +* Enable openpilot longitudinal control for Ford Q3 vehicles +* New Toyota TSS2 longitudinal tune +* Coming soon + * New driving model with gas gating + * Training data upload mode + +Version 0.9.7 (2024-06-13) ======================== * New driving model * Inputs the past curvature for smoother and more accurate lateral control * Simplified neural network architecture in the model's last layers * Minor fixes to desire augmentation and weight decay +* New driver monitoring model + * Improved end-to-end bit for phone detection * Adjust driving personality with the follow distance button -* Added toggle to enable driver monitoring even when openpilot is not engaged * Support for hybrid variants of supported Ford models * Fingerprinting without the OBD-II port on all cars +* Improved fuzzy fingerprinting for Ford and Volkswagen Version 0.9.6 (2024-02-27) ======================== diff --git a/SConstruct b/SConstruct index e827d66413..57831647cd 100644 --- a/SConstruct +++ b/SConstruct @@ -49,10 +49,6 @@ AddOption('--ccflags', default='', help='pass arbitrary flags over the command line') -AddOption('--snpe', - action='store_true', - help='use SNPE on PC') - AddOption('--external-sconscript', action='store', metavar='FILE', @@ -64,6 +60,10 @@ AddOption('--pc-thneed', dest='pc_thneed', help='use thneed on pc') +AddOption('--mutation', + action='store_true', + help='generate mutation-ready code') + AddOption('--minimal', action='store_false', dest='extras', @@ -102,7 +102,6 @@ if arch == "larch64": libpath = [ "/usr/local/lib", - "/usr/lib", "/system/vendor/lib64", f"#third_party/acados/{arch}/lib", ] @@ -169,10 +168,6 @@ else: if arch != "Darwin": ldflags += ["-Wl,--as-needed", "-Wl,--no-undefined"] -# Enable swaglog include in submodules -cflags += ['-DSWAGLOG="\\"common/swaglog.h\\""'] -cxxflags += ['-DSWAGLOG="\\"common/swaglog.h\\""'] - ccflags_option = GetOption('ccflags') if ccflags_option: ccflags += ccflags_option.split(' ') @@ -187,12 +182,9 @@ env = Environment( "-Werror", "-Wshadow", "-Wno-unknown-warning-option", - "-Wno-deprecated-register", - "-Wno-register", "-Wno-inconsistent-missing-override", "-Wno-c99-designator", "-Wno-reorder-init-list", - "-Wno-error=unused-but-set-variable", "-Wno-vla-cxx-extension", ] + cflags + ccflags, @@ -206,13 +198,8 @@ env = Environment( "#third_party/json11", "#third_party/linux/include", "#third_party/snpe/include", - "#third_party/qrcode", "#third_party", - "#cereal", "#msgq", - "#opendbc/can", - "#third_party/maplibre-native-qt/include", - f"#third_party/maplibre-native-qt/{arch}/include" ], CC='clang', @@ -224,10 +211,8 @@ env = Environment( CFLAGS=["-std=gnu11"] + cflags, CXXFLAGS=["-std=c++1z"] + cxxflags, LIBPATH=libpath + [ - "#cereal", - "#msgq", + "#msgq_repo", "#third_party", - "#opendbc/can", "#selfdrive/pandad", "#common", "#rednose/helpers", @@ -236,7 +221,7 @@ env = Environment( COMPILATIONDB_USE_ABSPATH=True, REDNOSE_ROOT="#", tools=["default", "cython", "compilation_db", "rednose_filter"], - toolpath=["#rednose_repo/site_scons/site_tools"], + toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"], ) if arch == "Darwin": @@ -275,11 +260,12 @@ if arch == "Darwin": else: envCython["LINKFLAGS"] = ["-pthread", "-shared"] -Export('envCython') +np_version = SCons.Script.Value(np.__version__) +Export('envCython', 'np_version') # Qt build environment qt_env = env.Clone() -qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "Multimedia", "Quick", "Qml", "QuickWidgets", "Location", "Positioning", "DBus", "Xml"] +qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "DBus", "Xml"] qt_libs = [] if arch == "Darwin": @@ -319,21 +305,17 @@ try: except SCons.Errors.UserError: qt_env.Tool('qt') -qt_env['CPPPATH'] += qt_dirs# + ["#selfdrive/ui/qt/"] +qt_env['CPPPATH'] += qt_dirs + ["#third_party/qrcode"] qt_flags = [ "-D_REENTRANT", "-DQT_NO_DEBUG", "-DQT_WIDGETS_LIB", "-DQT_GUI_LIB", - "-DQT_QUICK_LIB", - "-DQT_QUICKWIDGETS_LIB", - "-DQT_QML_LIB", "-DQT_CORE_LIB", "-DQT_MESSAGELOGCONTEXT", ] qt_env['CXXFLAGS'] += qt_flags -qt_env['LIBPATH'] += ['#selfdrive/ui', f"#third_party/maplibre-native-qt/{arch}/lib"] -qt_env['RPATH'] += [Dir(f"#third_party/maplibre-native-qt/{arch}/lib").srcnode().abspath] +qt_env['LIBPATH'] += ['#selfdrive/ui', ] qt_env['LIBS'] = qt_libs if GetOption("clazy"): @@ -353,27 +335,34 @@ Export('env', 'qt_env', 'arch', 'real_arch') SConscript(['common/SConscript']) Import('_common', '_gpucommon') -common = [_common, 'json11'] +common = [_common, 'json11', 'zmq'] gpucommon = [_gpucommon] Export('common', 'gpucommon') -# Build cereal and messaging -SConscript(['msgq/SConscript']) +# Build messaging (cereal + msgq + socketmaster + their dependencies) +# Enable swaglog include in submodules +env_swaglog = env.Clone() +env_swaglog['CXXFLAGS'].append('-DSWAGLOG="\\"common/swaglog.h\\""') +SConscript(['msgq_repo/SConscript'], exports={'env': env_swaglog}) +SConscript(['opendbc_repo/SConscript'], exports={'env': env_swaglog}) + SConscript(['cereal/SConscript']) +Import('socketmaster', 'msgq') +messaging = [socketmaster, msgq, 'zmq', 'capnp', 'kj',] +Export('messaging') + + # Build other submodules -SConscript([ - 'body/board/SConscript', - 'opendbc/can/SConscript', - 'panda/SConscript', -]) +SConscript(['panda/SConscript']) # Build rednose library SConscript(['rednose/SConscript']) # Build system services SConscript([ + 'system/ui/SConscript', 'system/proclogd/SConscript', 'system/ubloxd/SConscript', 'system/loggerd/SConscript', diff --git a/body b/body deleted file mode 160000 index 0e74db67ae..0000000000 --- a/body +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0e74db67ae6aaa7c30054bd4335dcafe69a5aa72 diff --git a/cereal/README.md b/cereal/README.md index e3326aab0e..ad940facdc 100644 --- a/cereal/README.md +++ b/cereal/README.md @@ -1,11 +1,6 @@ -# What is cereal? [![cereal tests](https://github.com/commaai/cereal/workflows/tests/badge.svg?event=push)](https://github.com/commaai/cereal/actions) [![codecov](https://codecov.io/gh/commaai/cereal/branch/master/graph/badge.svg)](https://codecov.io/gh/commaai/cereal) +# What is cereal? -cereal is both a messaging spec for robotics systems as well as generic high performance IPC pub sub messaging with a single publisher and multiple subscribers. - -Imagine this use case: -* A sensor process reads gyro measurements directly from an IMU and publishes a `sensorEvents` packet -* A calibration process subscribes to the `sensorEvents` packet to use the IMU -* A localization process subscribes to the `sensorEvents` packet to use the IMU also +cereal is the messaging system for openpilot. It uses [msgq](https://github.com/commaai/msgq) as a pub/sub backend, and [Cap'n proto](https://capnproto.org/capnp-tool.html) for serialization of the structs. ## Messaging Spec @@ -32,11 +27,7 @@ Forks of [openpilot](https://github.com/commaai/openpilot) might want to add thi spec, however this could conflict with future changes made in mainline cereal/openpilot. Rebasing against mainline openpilot then means breaking backwards-compatibility with all old logs of your fork. So we added reserved events in [custom.capnp](custom.capnp) that we will leave empty in mainline cereal/openpilot. **If you only modify those, you can ensure your -fork will remain backwards-compatible with all versions of mainline cereal/openpilot and your fork.** - -## Pub Sub Backends - -cereal supports two backends, one based on [zmq](https://zeromq.org/) and another called [msgq](messaging/msgq.cc), a custom pub sub based on shared memory that doesn't require the bytes to pass through the kernel. +fork will remain backwards-compatible with all versions of mainline openpilot and your fork.** Example --- diff --git a/cereal/SConscript b/cereal/SConscript index 5a71abbb81..a58a9490ce 100644 --- a/cereal/SConscript +++ b/cereal/SConscript @@ -1,31 +1,20 @@ -Import('env', 'envCython', 'arch', 'common', 'messaging') - -import shutil +Import('env', 'common', 'msgq') cereal_dir = Dir('.') gen_dir = Dir('gen') -other_dir = Dir('#msgq/messaging') # Build cereal schema_files = ['log.capnp', 'car.capnp', 'legacy.capnp', 'custom.capnp'] -env.Command(["gen/c/include/c++.capnp.h"], [], "mkdir -p " + gen_dir.path + "/c/include && touch $TARGETS") env.Command([f'gen/cpp/{s}.c++' for s in schema_files] + [f'gen/cpp/{s}.h' for s in schema_files], schema_files, f"capnpc --src-prefix={cereal_dir.path} $SOURCES -o c++:{gen_dir.path}/cpp/") -# TODO: remove non shared cereal and messaging -cereal_objects = env.SharedObject([f'gen/cpp/{s}.c++' for s in schema_files]) - -cereal = env.Library('cereal', cereal_objects) -env.SharedLibrary('cereal_shared', cereal_objects) +cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in schema_files]) # Build messaging - services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET') -env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=[messaging, 'zmq', common]) - +env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc'], LIBS=[msgq, common, 'pthread']) -socketmaster = env.SharedObject(['messaging/socketmaster.cc']) -socketmaster = env.Library('socketmaster', socketmaster) +socketmaster = env.Library('socketmaster', ['messaging/socketmaster.cc']) Export('cereal', 'socketmaster') diff --git a/cereal/car.capnp b/cereal/car.capnp deleted file mode 100644 index 8d76f0ead8..0000000000 --- a/cereal/car.capnp +++ /dev/null @@ -1,706 +0,0 @@ -using Cxx = import "./include/c++.capnp"; -$Cxx.namespace("cereal"); - -@0x8e2af1e708af8b8d; - -# ******* events causing controls state machine transition ******* - -struct CarEvent @0x9b1657f34caf3ad3 { - name @0 :EventName; - - # event types - enable @1 :Bool; - noEntry @2 :Bool; - warning @3 :Bool; # alerts presented only when enabled or soft disabling - userDisable @4 :Bool; - softDisable @5 :Bool; - immediateDisable @6 :Bool; - preEnable @7 :Bool; - permanent @8 :Bool; # alerts presented regardless of openpilot state - overrideLateral @10 :Bool; - overrideLongitudinal @9 :Bool; - - enum EventName @0xbaa8c5d505f727de { - canError @0; - steerUnavailable @1; - wrongGear @4; - doorOpen @5; - seatbeltNotLatched @6; - espDisabled @7; - wrongCarMode @8; - steerTempUnavailable @9; - reverseGear @10; - buttonCancel @11; - buttonEnable @12; - pedalPressed @13; # exits active state - preEnableStandstill @73; # added during pre-enable state with brake - gasPressedOverride @108; # added when user is pressing gas with no disengage on gas - steerOverride @114; - cruiseDisabled @14; - speedTooLow @17; - outOfSpace @18; - overheat @19; - calibrationIncomplete @20; - calibrationInvalid @21; - calibrationRecalibrating @117; - controlsMismatch @22; - pcmEnable @23; - pcmDisable @24; - radarFault @26; - brakeHold @28; - parkBrake @29; - manualRestart @30; - lowSpeedLockout @31; - joystickDebug @34; - steerTempUnavailableSilent @35; - resumeRequired @36; - preDriverDistracted @37; - promptDriverDistracted @38; - driverDistracted @39; - preDriverUnresponsive @43; - promptDriverUnresponsive @44; - driverUnresponsive @45; - belowSteerSpeed @46; - lowBattery @48; - accFaulted @51; - sensorDataInvalid @52; - commIssue @53; - commIssueAvgFreq @109; - tooDistracted @54; - posenetInvalid @55; - soundsUnavailable @56; - preLaneChangeLeft @57; - preLaneChangeRight @58; - laneChange @59; - lowMemory @63; - stockAeb @64; - ldw @65; - carUnrecognized @66; - invalidLkasSetting @69; - speedTooHigh @70; - laneChangeBlocked @71; - relayMalfunction @72; - stockFcw @74; - startup @75; - startupNoCar @76; - startupNoControl @77; - startupMaster @78; - startupNoFw @104; - fcw @79; - steerSaturated @80; - belowEngageSpeed @84; - noGps @85; - wrongCruiseMode @87; - modeldLagging @89; - deviceFalling @90; - fanMalfunction @91; - cameraMalfunction @92; - cameraFrameRate @110; - processNotRunning @95; - dashcamMode @96; - controlsInitializing @98; - usbError @99; - roadCameraError @100; - driverCameraError @101; - wideRoadCameraError @102; - highCpuUsage @105; - cruiseMismatch @106; - lkasDisabled @107; - canBusMissing @111; - controlsdLagging @112; - resumeBlocked @113; - steerTimeLimit @115; - vehicleSensorsInvalid @116; - locationdTemporaryError @103; - locationdPermanentError @118; - paramsdTemporaryError @50; - paramsdPermanentError @119; - actuatorsApiUnavailable @120; - - radarCanErrorDEPRECATED @15; - communityFeatureDisallowedDEPRECATED @62; - radarCommIssueDEPRECATED @67; - driverMonitorLowAccDEPRECATED @68; - gasUnavailableDEPRECATED @3; - dataNeededDEPRECATED @16; - modelCommIssueDEPRECATED @27; - ipasOverrideDEPRECATED @33; - geofenceDEPRECATED @40; - driverMonitorOnDEPRECATED @41; - driverMonitorOffDEPRECATED @42; - calibrationProgressDEPRECATED @47; - invalidGiraffeHondaDEPRECATED @49; - invalidGiraffeToyotaDEPRECATED @60; - internetConnectivityNeededDEPRECATED @61; - whitePandaUnsupportedDEPRECATED @81; - commIssueWarningDEPRECATED @83; - focusRecoverActiveDEPRECATED @86; - neosUpdateRequiredDEPRECATED @88; - modelLagWarningDEPRECATED @93; - startupOneplusDEPRECATED @82; - startupFuzzyFingerprintDEPRECATED @97; - noTargetDEPRECATED @25; - brakeUnavailableDEPRECATED @2; - plannerErrorDEPRECATED @32; - gpsMalfunctionDEPRECATED @94; - } -} - -# ******* main car state @ 100hz ******* -# all speeds in m/s - -struct CarState { - events @13 :List(CarEvent); - - # CAN health - canValid @26 :Bool; # invalid counter/checksums - canTimeout @40 :Bool; # CAN bus dropped out - canErrorCounter @48 :UInt32; - - # car speed - vEgo @1 :Float32; # best estimate of speed - aEgo @16 :Float32; # best estimate of acceleration - vEgoRaw @17 :Float32; # unfiltered speed from CAN sensors - vEgoCluster @44 :Float32; # best estimate of speed shown on car's instrument cluster, used for UI - - yawRate @22 :Float32; # best estimate of yaw rate - standstill @18 :Bool; - wheelSpeeds @2 :WheelSpeeds; - - # gas pedal, 0.0-1.0 - gas @3 :Float32; # this is user pedal only - gasPressed @4 :Bool; # this is user pedal only - - engineRpm @46 :Float32; - - # brake pedal, 0.0-1.0 - brake @5 :Float32; # this is user pedal only - brakePressed @6 :Bool; # this is user pedal only - regenBraking @45 :Bool; # this is user pedal only - parkingBrake @39 :Bool; - brakeHoldActive @38 :Bool; - - # steering wheel - steeringAngleDeg @7 :Float32; - steeringAngleOffsetDeg @37 :Float32; # Offset betweens sensors in case there multiple - steeringRateDeg @15 :Float32; - steeringTorque @8 :Float32; # TODO: standardize units - steeringTorqueEps @27 :Float32; # TODO: standardize units - steeringPressed @9 :Bool; # if the user is using the steering wheel - steerFaultTemporary @35 :Bool; # temporary EPS fault - steerFaultPermanent @36 :Bool; # permanent EPS fault - stockAeb @30 :Bool; - stockFcw @31 :Bool; - espDisabled @32 :Bool; - accFaulted @42 :Bool; - carFaultedNonCritical @47 :Bool; # some ECU is faulted, but car remains controllable - - # cruise state - cruiseState @10 :CruiseState; - - # gear - gearShifter @14 :GearShifter; - - # button presses - buttonEvents @11 :List(ButtonEvent); - leftBlinker @20 :Bool; - rightBlinker @21 :Bool; - genericToggle @23 :Bool; - - # lock info - doorOpen @24 :Bool; - seatbeltUnlatched @25 :Bool; - - # clutch (manual transmission only) - clutchPressed @28 :Bool; - - # blindspot sensors - leftBlindspot @33 :Bool; # Is there something blocking the left lane change - rightBlindspot @34 :Bool; # Is there something blocking the right lane change - - fuelGauge @41 :Float32; # battery or fuel tank level from 0.0 to 1.0 - charging @43 :Bool; - - # process meta - cumLagMs @50 :Float32; - - struct WheelSpeeds { - # optional wheel speeds - fl @0 :Float32; - fr @1 :Float32; - rl @2 :Float32; - rr @3 :Float32; - } - - struct CruiseState { - enabled @0 :Bool; - speed @1 :Float32; - speedCluster @6 :Float32; # Set speed as shown on instrument cluster - available @2 :Bool; - speedOffset @3 :Float32; - standstill @4 :Bool; - nonAdaptive @5 :Bool; - } - - enum GearShifter { - unknown @0; - park @1; - drive @2; - neutral @3; - reverse @4; - sport @5; - low @6; - brake @7; - eco @8; - manumatic @9; - } - - # send on change - struct ButtonEvent { - pressed @0 :Bool; - type @1 :Type; - - enum Type { - unknown @0; - leftBlinker @1; - rightBlinker @2; - accelCruise @3; - decelCruise @4; - cancel @5; - altButton1 @6; - altButton2 @7; - altButton3 @8; - setCruise @9; - resumeCruise @10; - gapAdjustCruise @11; - } - } - - # deprecated - errorsDEPRECATED @0 :List(CarEvent.EventName); - brakeLightsDEPRECATED @19 :Bool; - steeringRateLimitedDEPRECATED @29 :Bool; - canMonoTimesDEPRECATED @12: List(UInt64); - canRcvTimeoutDEPRECATED @49 :Bool; -} - -# ******* radar state @ 20hz ******* - -struct RadarData @0x888ad6581cf0aacb { - errors @0 :List(Error); - points @1 :List(RadarPoint); - - enum Error { - canError @0; - fault @1; - wrongConfig @2; - } - - # similar to LiveTracks - # is one timestamp valid for all? I think so - struct RadarPoint { - trackId @0 :UInt64; # no trackId reuse - - # these 3 are the minimum required - dRel @1 :Float32; # m from the front bumper of the car - yRel @2 :Float32; # m - vRel @3 :Float32; # m/s - - # these are optional and valid if they are not NaN - aRel @4 :Float32; # m/s^2 - yvRel @5 :Float32; # m/s - - # some radars flag measurements VS estimates - measured @6 :Bool; - } - - # deprecated - canMonoTimesDEPRECATED @2 :List(UInt64); -} - -# ******* car controls @ 100hz ******* - -struct CarControl { - # must be true for any actuator commands to work - enabled @0 :Bool; - latActive @11: Bool; - longActive @12: Bool; - - # Actuator commands as computed by controlsd - actuators @6 :Actuators; - - # moved to CarOutput - actuatorsOutputDEPRECATED @10 :Actuators; - - leftBlinker @15: Bool; - rightBlinker @16: Bool; - - orientationNED @13 :List(Float32); - angularVelocity @14 :List(Float32); - - cruiseControl @4 :CruiseControl; - hudControl @5 :HUDControl; - - struct Actuators { - # range from 0.0 - 1.0 - gas @0: Float32; - brake @1: Float32; - # range from -1.0 - 1.0 - steer @2: Float32; - # value sent over can to the car - steerOutputCan @8: Float32; - steeringAngleDeg @3: Float32; - - curvature @7: Float32; - - speed @6: Float32; # m/s - accel @4: Float32; # m/s^2 - longControlState @5: LongControlState; - - enum LongControlState @0xe40f3a917d908282{ - off @0; - pid @1; - stopping @2; - starting @3; - } - } - - struct CruiseControl { - cancel @0: Bool; - resume @1: Bool; - override @4: Bool; - speedOverrideDEPRECATED @2: Float32; - accelOverrideDEPRECATED @3: Float32; - } - - struct HUDControl { - speedVisible @0: Bool; - setSpeed @1: Float32; - lanesVisible @2: Bool; - leadVisible @3: Bool; - visualAlert @4: VisualAlert; - audibleAlert @5: AudibleAlert; - rightLaneVisible @6: Bool; - leftLaneVisible @7: Bool; - rightLaneDepart @8: Bool; - leftLaneDepart @9: Bool; - leadDistanceBars @10: Int8; # 1-3: 1 is closest, 3 is farthest. some ports may utilize 2-4 bars instead - - enum VisualAlert { - # these are the choices from the Honda - # map as good as you can for your car - none @0; - fcw @1; - steerRequired @2; - brakePressed @3; - wrongGear @4; - seatbeltUnbuckled @5; - speedTooHigh @6; - ldw @7; - } - - enum AudibleAlert { - none @0; - - engage @1; - disengage @2; - refuse @3; - - warningSoft @4; - warningImmediate @5; - - prompt @6; - promptRepeat @7; - promptDistracted @8; - } - } - - gasDEPRECATED @1 :Float32; - brakeDEPRECATED @2 :Float32; - steeringTorqueDEPRECATED @3 :Float32; - activeDEPRECATED @7 :Bool; - rollDEPRECATED @8 :Float32; - pitchDEPRECATED @9 :Float32; -} - -struct CarOutput { - # Any car specific rate limits or quirks applied by - # the CarController are reflected in actuatorsOutput - # and matches what is sent to the car - actuatorsOutput @0 :CarControl.Actuators; -} - -# ****** car param ****** - -struct CarParams { - carName @0 :Text; - carFingerprint @1 :Text; - fuzzyFingerprint @55 :Bool; - - notCar @66 :Bool; # flag for non-car robotics platforms - - pcmCruise @3 :Bool; # is openpilot's state tied to the PCM's cruise state? - enableDsu @5 :Bool; # driving support unit - enableBsm @56 :Bool; # blind spot monitoring - flags @64 :UInt32; # flags for car specific quirks - experimentalLongitudinalAvailable @71 :Bool; - - minEnableSpeed @7 :Float32; - minSteerSpeed @8 :Float32; - safetyConfigs @62 :List(SafetyConfig); - alternativeExperience @65 :Int16; # panda flag for features like no disengage on gas - - # Car docs fields - maxLateralAccel @68 :Float32; - autoResumeSng @69 :Bool; # describes whether car can resume from a stop automatically - - # things about the car in the manual - mass @17 :Float32; # [kg] curb weight: all fluids no cargo - wheelbase @18 :Float32; # [m] distance from rear axle to front axle - centerToFront @19 :Float32; # [m] distance from center of mass to front axle - steerRatio @20 :Float32; # [] ratio of steering wheel angle to front wheel angle - steerRatioRear @21 :Float32; # [] ratio of steering wheel angle to rear wheel angle (usually 0) - - # things we can derive - rotationalInertia @22 :Float32; # [kg*m2] body rotational inertia - tireStiffnessFactor @72 :Float32; # scaling factor used in calculating tireStiffness[Front,Rear] - tireStiffnessFront @23 :Float32; # [N/rad] front tire coeff of stiff - tireStiffnessRear @24 :Float32; # [N/rad] rear tire coeff of stiff - - longitudinalTuning @25 :LongitudinalPIDTuning; - lateralParams @48 :LateralParams; - lateralTuning :union { - pid @26 :LateralPIDTuning; - indiDEPRECATED @27 :LateralINDITuning; - lqrDEPRECATED @40 :LateralLQRTuning; - torque @67 :LateralTorqueTuning; - } - - steerLimitAlert @28 :Bool; - steerLimitTimer @47 :Float32; # time before steerLimitAlert is issued - - vEgoStopping @29 :Float32; # Speed at which the car goes into stopping state - vEgoStarting @59 :Float32; # Speed at which the car goes into starting state - stoppingControl @31 :Bool; # Does the car allow full control even at lows speeds when stopping - steerControlType @34 :SteerControlType; - radarUnavailable @35 :Bool; # True when radar objects aren't visible on CAN or aren't parsed out - stopAccel @60 :Float32; # Required acceleration to keep vehicle stationary - stoppingDecelRate @52 :Float32; # m/s^2/s while trying to stop - startAccel @32 :Float32; # Required acceleration to get car moving - startingState @70 :Bool; # Does this car make use of special starting state - - steerActuatorDelay @36 :Float32; # Steering wheel actuator delay in seconds - longitudinalActuatorDelayLowerBound @61 :Float32; # Gas/Brake actuator delay in seconds, lower bound - longitudinalActuatorDelayUpperBound @58 :Float32; # Gas/Brake actuator delay in seconds, upper bound - openpilotLongitudinalControl @37 :Bool; # is openpilot doing the longitudinal control? - carVin @38 :Text; # VIN number queried during fingerprinting - dashcamOnly @41: Bool; - passive @73: Bool; # is openpilot in control? - transmissionType @43 :TransmissionType; - carFw @44 :List(CarFw); - - radarTimeStep @45: Float32 = 0.05; # time delta between radar updates, 20Hz is very standard - fingerprintSource @49: FingerprintSource; - networkLocation @50 :NetworkLocation; # Where Panda/C2 is integrated into the car's CAN network - - wheelSpeedFactor @63 :Float32; # Multiplier on wheels speeds to computer actual speeds - - struct SafetyConfig { - safetyModel @0 :SafetyModel; - safetyParam @3 :UInt16; - safetyParamDEPRECATED @1 :Int16; - safetyParam2DEPRECATED @2 :UInt32; - } - - struct LateralParams { - torqueBP @0 :List(Int32); - torqueV @1 :List(Int32); - } - - struct LateralPIDTuning { - kpBP @0 :List(Float32); - kpV @1 :List(Float32); - kiBP @2 :List(Float32); - kiV @3 :List(Float32); - kf @4 :Float32; - } - - struct LateralTorqueTuning { - useSteeringAngle @0 :Bool; - kp @1 :Float32; - ki @2 :Float32; - friction @3 :Float32; - kf @4 :Float32; - steeringAngleDeadzoneDeg @5 :Float32; - latAccelFactor @6 :Float32; - latAccelOffset @7 :Float32; - } - - struct LongitudinalPIDTuning { - kpBP @0 :List(Float32); - kpV @1 :List(Float32); - kiBP @2 :List(Float32); - kiV @3 :List(Float32); - kf @6 :Float32; - deadzoneBP @4 :List(Float32); - deadzoneV @5 :List(Float32); - } - - struct LateralINDITuning { - outerLoopGainBP @4 :List(Float32); - outerLoopGainV @5 :List(Float32); - innerLoopGainBP @6 :List(Float32); - innerLoopGainV @7 :List(Float32); - timeConstantBP @8 :List(Float32); - timeConstantV @9 :List(Float32); - actuatorEffectivenessBP @10 :List(Float32); - actuatorEffectivenessV @11 :List(Float32); - - outerLoopGainDEPRECATED @0 :Float32; - innerLoopGainDEPRECATED @1 :Float32; - timeConstantDEPRECATED @2 :Float32; - actuatorEffectivenessDEPRECATED @3 :Float32; - } - - struct LateralLQRTuning { - scale @0 :Float32; - ki @1 :Float32; - dcGain @2 :Float32; - - # State space system - a @3 :List(Float32); - b @4 :List(Float32); - c @5 :List(Float32); - - k @6 :List(Float32); # LQR gain - l @7 :List(Float32); # Kalman gain - } - - enum SafetyModel { - silent @0; - hondaNidec @1; - toyota @2; - elm327 @3; - gm @4; - hondaBoschGiraffe @5; - ford @6; - cadillac @7; - hyundai @8; - chrysler @9; - tesla @10; - subaru @11; - gmPassive @12; - mazda @13; - nissan @14; - volkswagen @15; - toyotaIpas @16; - allOutput @17; - gmAscm @18; - noOutput @19; # like silent but without silent CAN TXs - hondaBosch @20; - volkswagenPq @21; - subaruPreglobal @22; # pre-Global platform - hyundaiLegacy @23; - hyundaiCommunity @24; - volkswagenMlb @25; - hongqi @26; - body @27; - hyundaiCanfd @28; - volkswagenMqbEvo @29; - chryslerCusw @30; - psa @31; - } - - enum SteerControlType { - torque @0; - angle @1; - - curvatureDEPRECATED @2; - } - - enum TransmissionType { - unknown @0; - automatic @1; # Traditional auto, including DSG - manual @2; # True "stick shift" only - direct @3; # Electric vehicle or other direct drive - cvt @4; - } - - struct CarFw { - ecu @0 :Ecu; - fwVersion @1 :Data; - address @2 :UInt32; - subAddress @3 :UInt8; - responseAddress @4 :UInt32; - request @5 :List(Data); - brand @6 :Text; - bus @7 :UInt8; - logging @8 :Bool; - obdMultiplexing @9 :Bool; - } - - enum Ecu { - eps @0; - abs @1; - fwdRadar @2; - fwdCamera @3; - engine @4; - unknown @5; - transmission @8; # Transmission Control Module - hybrid @18; # hybrid control unit, e.g. Chrysler's HCP, Honda's IMA Control Unit, Toyota's hybrid control computer - srs @9; # airbag - gateway @10; # can gateway - hud @11; # heads up display - combinationMeter @12; # instrument cluster - electricBrakeBooster @15; - shiftByWire @16; - adas @19; - cornerRadar @21; - hvac @20; - parkingAdas @7; # parking assist system ECU, e.g. Toyota's IPAS, Hyundai's RSPA, etc. - epb @22; # electronic parking brake - telematics @23; - body @24; # body control module - - # Toyota only - dsu @6; - - # Honda only - vsa @13; # Vehicle Stability Assist - programmedFuelInjection @14; - - debug @17; - } - - enum FingerprintSource { - can @0; - fw @1; - fixed @2; - } - - enum NetworkLocation { - fwdCamera @0; # Standard/default integration at LKAS camera - gateway @1; # Integration at vehicle's CAN gateway - } - - enableGasInterceptorDEPRECATED @2 :Bool; - enableCameraDEPRECATED @4 :Bool; - enableApgsDEPRECATED @6 :Bool; - steerRateCostDEPRECATED @33 :Float32; - isPandaBlackDEPRECATED @39 :Bool; - hasStockCameraDEPRECATED @57 :Bool; - safetyParamDEPRECATED @10 :Int16; - safetyModelDEPRECATED @9 :SafetyModel; - safetyModelPassiveDEPRECATED @42 :SafetyModel = silent; - minSpeedCanDEPRECATED @51 :Float32; - communityFeatureDEPRECATED @46: Bool; - startingAccelRateDEPRECATED @53 :Float32; - steerMaxBPDEPRECATED @11 :List(Float32); - steerMaxVDEPRECATED @12 :List(Float32); - gasMaxBPDEPRECATED @13 :List(Float32); - gasMaxVDEPRECATED @14 :List(Float32); - brakeMaxBPDEPRECATED @15 :List(Float32); - brakeMaxVDEPRECATED @16 :List(Float32); - directAccelControlDEPRECATED @30 :Bool; - maxSteeringAngleDegDEPRECATED @54 :Float32; -} diff --git a/cereal/car.capnp b/cereal/car.capnp new file mode 120000 index 0000000000..4bc7f89b1f --- /dev/null +++ b/cereal/car.capnp @@ -0,0 +1 @@ +../opendbc_repo/opendbc/car/car.capnp \ No newline at end of file diff --git a/cereal/log.capnp b/cereal/log.capnp index 36ca692dcc..83e64b5a85 100644 --- a/cereal/log.capnp +++ b/cereal/log.capnp @@ -17,6 +17,119 @@ struct Map(Key, Value) { } } +struct OnroadEvent @0xc4fa6047f024e718 { + name @0 :EventName; + + # event types + enable @1 :Bool; + noEntry @2 :Bool; + warning @3 :Bool; # alerts presented only when enabled or soft disabling + userDisable @4 :Bool; + softDisable @5 :Bool; + immediateDisable @6 :Bool; + preEnable @7 :Bool; + permanent @8 :Bool; # alerts presented regardless of openpilot state + overrideLateral @10 :Bool; + overrideLongitudinal @9 :Bool; + + enum EventName @0x91f1992a1f77fb03 { + canError @0; + steerUnavailable @1; + wrongGear @2; + doorOpen @3; + seatbeltNotLatched @4; + espDisabled @5; + wrongCarMode @6; + steerTempUnavailable @7; + reverseGear @8; + buttonCancel @9; + buttonEnable @10; + pedalPressed @11; # exits active state + preEnableStandstill @12; # added during pre-enable state with brake + gasPressedOverride @13; # added when user is pressing gas with no disengage on gas + steerOverride @14; + cruiseDisabled @15; + speedTooLow @16; + outOfSpace @17; + overheat @18; + calibrationIncomplete @19; + calibrationInvalid @20; + calibrationRecalibrating @21; + controlsMismatch @22; + pcmEnable @23; + pcmDisable @24; + radarFault @25; + brakeHold @26; + parkBrake @27; + manualRestart @28; + joystickDebug @29; + longitudinalManeuver @30; + steerTempUnavailableSilent @31; + resumeRequired @32; + preDriverDistracted @33; + promptDriverDistracted @34; + driverDistracted @35; + preDriverUnresponsive @36; + promptDriverUnresponsive @37; + driverUnresponsive @38; + belowSteerSpeed @39; + lowBattery @40; + accFaulted @41; + sensorDataInvalid @42; + commIssue @43; + commIssueAvgFreq @44; + tooDistracted @45; + posenetInvalid @46; + preLaneChangeLeft @48; + preLaneChangeRight @49; + laneChange @50; + lowMemory @51; + stockAeb @52; + ldw @53; + carUnrecognized @54; + invalidLkasSetting @55; + speedTooHigh @56; + laneChangeBlocked @57; + relayMalfunction @58; + stockFcw @59; + startup @60; + startupNoCar @61; + startupNoControl @62; + startupNoSecOcKey @63; + startupMaster @64; + fcw @65; + steerSaturated @66; + belowEngageSpeed @67; + noGps @68; + wrongCruiseMode @69; + modeldLagging @70; + deviceFalling @71; + fanMalfunction @72; + cameraMalfunction @73; + cameraFrameRate @74; + processNotRunning @75; + dashcamMode @76; + selfdriveInitializing @77; + usbError @78; + cruiseMismatch @79; + canBusMissing @80; + selfdrivedLagging @81; + resumeBlocked @82; + steerTimeLimit @83; + vehicleSensorsInvalid @84; + locationdTemporaryError @85; + locationdPermanentError @86; + paramsdTemporaryError @87; + paramsdPermanentError @88; + actuatorsApiUnavailable @89; + espActive @90; + personalityChanged @91; + aeb @92; + + soundsUnavailableDEPRECATED @47; + } +} + enum LongitudinalPersonality { aggressive @0; standard @1; @@ -137,8 +250,6 @@ struct FrameData { requestId @28 :UInt32; encodeId @1 :UInt32; - frameType @7 :FrameType; - # Timestamps timestampEof @2 :UInt64; timestampSof @8 :UInt64; @@ -158,7 +269,7 @@ struct FrameData { temperaturesC @24 :List(Float32); - enum FrameType { + enum FrameTypeDEPRECATED { unknown @0; neo @1; chffrAndroid @2; @@ -175,6 +286,7 @@ struct FrameData { frameLengthDEPRECATED @3 :Int32; globalGainDEPRECATED @5 :Int32; + frameTypeDEPRECATED @7 :FrameTypeDEPRECATED; androidCaptureResultDEPRECATED @9 :AndroidCaptureResult; lensPosDEPRECATED @11 :Int32; lensSagDEPRECATED @12 :Float32; @@ -297,6 +409,7 @@ struct GpsLocationData { speedAccuracy @12 :Float32; hasFix @13 :Bool; + satelliteCount @14 :Int8; enum SensorSource { android @0; @@ -337,9 +450,9 @@ enum LaneChangeDirection { struct CanData { address @0 :UInt32; - busTime @1 :UInt16; dat @2 :Data; src @3 :UInt8; + busTimeDEPRECATED @1 :UInt16; } struct DeviceState @0xa4d8b5af2aa492eb { @@ -374,6 +487,9 @@ struct DeviceState @0xa4d8b5af2aa492eb { nvmeTempC @35 :List(Float32); modemTempC @36 :List(Float32); pmicTempC @39 :List(Float32); + intakeTempC @46 :Float32; + exhaustTempC @47 :Float32; + caseTempC @48 :Float32; maxTempC @44 :Float32; # max of other temps, used to control fan thermalZones @38 :List(ThermalZone); thermalStatus @14 :ThermalStatus; @@ -612,7 +728,6 @@ struct RadarState @0x9a185389d6fdd05f { leadOne @3 :LeadData; leadTwo @4 :LeadData; - cumLagMs @5 :Float32; struct LeadData { dRel @0 :Float32; @@ -642,6 +757,7 @@ struct RadarState @0x9a185389d6fdd05f { calCycleDEPRECATED @8 :Int32; calPercDEPRECATED @9 :Int8; canMonoTimesDEPRECATED @10 :List(UInt64); + cumLagMsDEPRECATED @5 :Float32; } struct LiveCalibrationData { @@ -672,7 +788,7 @@ struct LiveCalibrationData { } } -struct LiveTracks { +struct LiveTracksDEPRECATED { trackId @0 :Int32; dRel @1 :Float32; yRel @2 :Float32; @@ -685,45 +801,61 @@ struct LiveTracks { oncoming @9 :Bool; } +struct SelfdriveState { + # high level system state + state @0 :OpenpilotState; + enabled @1 :Bool; + active @2 :Bool; + engageable @9 :Bool; # can OP be engaged? + + # UI alerts + alertText1 @3 :Text; + alertText2 @4 :Text; + alertStatus @5 :AlertStatus; + alertSize @6 :AlertSize; + alertType @7 :Text; + alertSound @8 :Car.CarControl.HUDControl.AudibleAlert; + alertHudVisual @12 :Car.CarControl.HUDControl.VisualAlert; + + # configurable driving settings + experimentalMode @10 :Bool; + personality @11 :LongitudinalPersonality; + + enum OpenpilotState @0xdbe58b96d2d1ac61 { + disabled @0; + preEnabled @1; + enabled @2; + softDisabling @3; + overriding @4; # superset of overriding with steering or accelerator + } + + enum AlertStatus @0xa0d0dcd113193c62 { + normal @0; + userPrompt @1; + critical @2; + } + + enum AlertSize @0xe98bb99d6e985f64 { + none @0; + small @1; + mid @2; + full @3; + } +} + struct ControlsState @0x97ff69c53601abf1 { - startMonoTime @48 :UInt64; longitudinalPlanMonoTime @28 :UInt64; lateralPlanMonoTime @50 :UInt64; - state @31 :OpenpilotState; - enabled @19 :Bool; - active @36 :Bool; - - experimentalMode @64 :Bool; - personality @66 :LongitudinalPersonality; - longControlState @30 :Car.CarControl.Actuators.LongControlState; - vPid @2 :Float32; - vTargetLead @3 :Float32; - vCruise @22 :Float32; # actual set speed - vCruiseCluster @63 :Float32; # set speed to display in the UI upAccelCmd @4 :Float32; uiAccelCmd @5 :Float32; ufAccelCmd @33 :Float32; - aTarget @35 :Float32; curvature @37 :Float32; # path curvature from vehicle model desiredCurvature @61 :Float32; # lag adjusted curvatures used by lateral controllers forceDecel @51 :Bool; - # UI alerts - alertText1 @24 :Text; - alertText2 @25 :Text; - alertStatus @38 :AlertStatus; - alertSize @39 :AlertSize; - alertBlinkingRate @42 :Float32; - alertType @44 :Text; - alertSound @56 :Car.CarControl.HUDControl.AudibleAlert; - engageable @41 :Bool; # can OP be engaged? - - cumLagMs @15 :Float32; - lateralControlState :union { - indiState @52 :LateralINDIState; pidState @53 :LateralPIDState; angleState @58 :LateralAngleState; debugState @59 :LateralDebugState; @@ -731,27 +863,7 @@ struct ControlsState @0x97ff69c53601abf1 { curvatureStateDEPRECATED @65 :LateralCurvatureState; lqrStateDEPRECATED @55 :LateralLQRState; - } - - enum OpenpilotState @0xdbe58b96d2d1ac61 { - disabled @0; - preEnabled @1; - enabled @2; - softDisabling @3; - overriding @4; # superset of overriding with steering or accelerator - } - - enum AlertStatus { - normal @0; # low priority alert for user's convenience - userPrompt @1; # mid priority alert that might require user intervention - critical @2; # high priority alert that needs immediate user intervention - } - - enum AlertSize { - none @0; # don't display the alert - small @1; # small box - mid @2; # mid screen - full @3; # full screen + indiStateDEPRECATED @52 :LateralINDIState; } struct LateralINDIState { @@ -866,6 +978,58 @@ struct ControlsState @0x97ff69c53601abf1 { canMonoTimesDEPRECATED @21 :List(UInt64); desiredCurvatureRateDEPRECATED @62 :Float32; canErrorCounterDEPRECATED @57 :UInt32; + vPidDEPRECATED @2 :Float32; + alertBlinkingRateDEPRECATED @42 :Float32; + alertText1DEPRECATED @24 :Text; + alertText2DEPRECATED @25 :Text; + alertStatusDEPRECATED @38 :SelfdriveState.AlertStatus; + alertSizeDEPRECATED @39 :SelfdriveState.AlertSize; + alertTypeDEPRECATED @44 :Text; + alertSound2DEPRECATED @56 :Car.CarControl.HUDControl.AudibleAlert; + engageableDEPRECATED @41 :Bool; # can OP be engaged? + stateDEPRECATED @31 :SelfdriveState.OpenpilotState; + enabledDEPRECATED @19 :Bool; + activeDEPRECATED @36 :Bool; + experimentalModeDEPRECATED @64 :Bool; + personalityDEPRECATED @66 :LongitudinalPersonality; + vCruiseDEPRECATED @22 :Float32; # actual set speed + vCruiseClusterDEPRECATED @63 :Float32; # set speed to display in the UI + startMonoTimeDEPRECATED @48 :UInt64; + cumLagMsDEPRECATED @15 :Float32; + aTargetDEPRECATED @35 :Float32; + vTargetLeadDEPRECATED @3 :Float32; +} + +struct DrivingModelData { + frameId @0 :UInt32; + frameIdExtra @1 :UInt32; + frameDropPerc @6 :Float32; + modelExecutionTime @7 :Float32; + + action @2 :ModelDataV2.Action; + + laneLineMeta @3 :LaneLineMeta; + meta @4 :MetaData; + + path @5 :PolyPath; + + struct PolyPath { + xCoefficients @0 :List(Float32); + yCoefficients @1 :List(Float32); + zCoefficients @2 :List(Float32); + } + + struct LaneLineMeta { + leftY @0 :Float32; + rightY @1 :Float32; + leftProb @2 :Float32; + rightProb @3 :Float32; + } + + struct MetaData { + laneChangeState @0 :LaneChangeState; + laneChangeDirection @1 :LaneChangeDirection; + } } # All SI units and in device frame @@ -886,7 +1050,6 @@ struct ModelDataV2 { frameDropPerc @2 :Float32; timestampEof @3 :UInt64; modelExecutionTime @15 :Float32; - gpuExecutionTime @17 :Float32; rawPredictions @16 :Data; # predicted future position, orientation, etc.. @@ -913,12 +1076,13 @@ struct ModelDataV2 { # Model perceived motion temporalPose @21 :Pose; + # e2e lateral planner + action @26: Action; + + gpuExecutionTimeDEPRECATED @17 :Float32; navEnabledDEPRECATED @22 :Bool; locationMonoTimeDEPRECATED @24 :UInt64; - - # e2e lateral planner lateralPlannerSolutionDEPRECATED @25: LateralPlannerSolution; - action @26: Action; struct LeadDataV2 { prob @0 :Float32; # probability that car is your lead at time t @@ -980,6 +1144,8 @@ struct ModelDataV2 { brake3MetersPerSecondSquaredProbs @4 :List(Float32); brake4MetersPerSecondSquaredProbs @5 :List(Float32); brake5MetersPerSecondSquaredProbs @6 :List(Float32); + gasPressProbs @7 :List(Float32); + brakePressProbs @8 :List(Float32); } struct Pose { @@ -1049,6 +1215,14 @@ struct AndroidLogEntry { message @6 :Text; } +struct DriverAssistance { + # Lane Departure Warnings + leftLaneDeparture @0 :Bool; + rightLaneDeparture @1 :Bool; + + # FCW, AEB, etc. will go here +} + struct LongitudinalPlan @0xe00b5b3eba12876c { modelMonoTime @9 :UInt64; hasLead @7 :Bool; @@ -1060,6 +1234,11 @@ struct LongitudinalPlan @0xe00b5b3eba12876c { accels @32 :List(Float32); speeds @33 :List(Float32); jerks @34 :List(Float32); + aTarget @18 :Float32; + shouldStop @37: Bool; + allowThrottle @38: Bool; + allowBrake @39: Bool; + solverExecutionTime @35 :Float32; @@ -1076,7 +1255,6 @@ struct LongitudinalPlan @0xe00b5b3eba12876c { aCruiseDEPRECATED @17 :Float32; vTargetDEPRECATED @3 :Float32; vTargetFutureDEPRECATED @14 :Float32; - aTargetDEPRECATED @18 :Float32; vStartDEPRECATED @26 :Float32; aStartDEPRECATED @27 :Float32; vMaxDEPRECATED @20 :Float32; @@ -1096,7 +1274,7 @@ struct LongitudinalPlan @0xe00b5b3eba12876c { radarValidDEPRECATED @28 :Bool; radarCanErrorDEPRECATED @30 :Bool; commIssueDEPRECATED @31 :Bool; - eventsDEPRECATED @13 :List(Car.CarEvent); + eventsDEPRECATED @13 :List(Car.OnroadEventDEPRECATED); gpsTrajectoryDEPRECATED @12 :GpsTrajectory; gpsPlannerActiveDEPRECATED @19 :Bool; personalityDEPRECATED @36 :LongitudinalPersonality; @@ -1217,6 +1395,46 @@ struct LiveLocationKalman { } } + +struct LivePose { + # More info on reference frames: + # https://github.com/commaai/openpilot/tree/master/common/transformations + orientationNED @0 :XYZMeasurement; + velocityDevice @1 :XYZMeasurement; + accelerationDevice @2 :XYZMeasurement; + angularVelocityDevice @3 :XYZMeasurement; + + inputsOK @4 :Bool = false; + posenetOK @5 :Bool = false; + sensorsOK @6 :Bool = false; + + debugFilterState @7 :FilterState; + + struct XYZMeasurement { + x @0 :Float32; + y @1 :Float32; + z @2 :Float32; + xStd @3 :Float32; + yStd @4 :Float32; + zStd @5 :Float32; + valid @6 :Bool; + } + + struct FilterState { + value @0 : List(Float64); + std @1 : List(Float64); + valid @2 : Bool; + + observations @3 :List(Observation); + + struct Observation { + kind @0 :Int32; + value @1 :List(Float32); + error @2 :List(Float32); + } + } +} + struct ProcLog { cpuTimes @0 :List(CPUTimes); mem @1 :Mem; @@ -1911,7 +2129,8 @@ struct Joystick { struct DriverStateV2 { frameId @0 :UInt32; modelExecutionTime @1 :Float32; - dspExecutionTime @2 :Float32; + dspExecutionTimeDEPRECATED @2 :Float32; + gpuExecutionTime @8 :Float32; rawPredictions @3 :Data; poorVisionProb @4 :Float32; @@ -1970,7 +2189,7 @@ struct DriverStateDEPRECATED @0xb83c6cc593ed0a00 { } struct DriverMonitoringState @0xb83cda094a1da284 { - events @0 :List(Car.CarEvent); + events @18 :List(OnroadEvent); faceDetected @1 :Bool; isDistracted @2 :Bool; distractedType @17 :UInt32; @@ -1989,6 +2208,7 @@ struct DriverMonitoringState @0xb83cda094a1da284 { isPreviewDEPRECATED @15 :Bool; rhdCheckedDEPRECATED @5 :Bool; + eventsDEPRECATED @0 :List(Car.OnroadEventDEPRECATED); } struct Boot { @@ -2016,9 +2236,15 @@ struct LiveParametersData { stiffnessFactorStd @12 :Float32; steerRatioStd @13 :Float32; roll @14 :Float32; - filterState @15 :LiveLocationKalman.Measurement; + debugFilterState @16 :FilterState; yawRateDEPRECATED @7 :Float32; + filterStateDEPRECATED @15 :LiveLocationKalman.Measurement; + + struct FilterState { + value @0 : List(Float64); + std @1 : List(Float64); + } } struct LiveTorqueParametersData { @@ -2198,6 +2424,11 @@ struct EncodeData { height @5 :UInt32; } +struct DebugAlert { + alertText1 @0 :Text; + alertText2 @1 :Text; +} + struct UserFlag { } @@ -2210,6 +2441,14 @@ struct Microphone { filteredSoundPressureWeightedDb @2 :Float32; } +struct Touch { + sec @0 :Int64; + usec @1 :Int64; + type @2 :UInt8; + code @3 :Int32; + value @4 :Int32; +} + struct Event { logMonoTime @0 :UInt64; # nanoseconds valid @67 :Bool = true; @@ -2226,6 +2465,7 @@ struct Event { gpsNMEA @3 :GPSNMEAData; can @5 :List(CanData); controlsState @7 :ControlsState; + selfdriveState @130 :SelfdriveState; gyroscope @99 :SensorEventData; gyroscope2 @100 :SensorEventData; accelerometer @98 :SensorEventData; @@ -2237,14 +2477,14 @@ struct Event { pandaStates @81 :List(PandaState); peripheralState @80 :PeripheralState; radarState @13 :RadarState; - liveTracks @16 :List(LiveTracks); + liveTracks @131 :Car.RadarData; sendcan @17 :List(CanData); liveCalibration @19 :LiveCalibrationData; carState @22 :Car.CarState; carControl @23 :Car.CarControl; carOutput @127 :Car.CarOutput; longitudinalPlan @24 :LongitudinalPlan; - uiPlan @106 :UiPlan; + driverAssistance @132 :DriverAssistance; ubloxGnss @34 :UbloxGnss; ubloxRaw @39 :Data; qcomGnss @31 :QcomGnss; @@ -2255,11 +2495,12 @@ struct Event { liveTorqueParameters @94 :LiveTorqueParametersData; cameraOdometry @63 :CameraOdometry; thumbnail @66: Thumbnail; - onroadEvents @68: List(Car.CarEvent); + onroadEvents @134: List(OnroadEvent); carParams @69: Car.CarParams; driverMonitoringState @71: DriverMonitoringState; - liveLocationKalman @72 :LiveLocationKalman; + livePose @129 :LivePose; modelV2 @75 :ModelDataV2; + drivingModelData @128 :DrivingModelData; driverStateV2 @92 :DriverStateV2; # camera stuff, each camera state has a matching encode idx @@ -2288,6 +2529,9 @@ struct Event { logMessage @18 :Text; errorLogMessage @85 :Text; + # touch frame + touch @135 :List(Touch); + # navigation navInstruction @82 :NavInstruction; navRoute @83 :NavRoute; @@ -2304,6 +2548,7 @@ struct Event { driverEncodeData @87 :EncodeData; wideRoadEncodeData @88 :EncodeData; qRoadEncodeData @89 :EncodeData; + alertDebug @133 :DebugAlert; livestreamRoadEncodeData @120 :EncodeData; livestreamWideRoadEncodeData @121 :EncodeData; @@ -2329,7 +2574,7 @@ struct Event { model @9 :Legacy.ModelData; # TODO: rename modelV2 and mark this as deprecated liveMpcDEPRECATED @36 :LiveMpcData; liveLongitudinalMpcDEPRECATED @37 :LiveLongitudinalMpcData; - liveLocationKalmanDEPRECATED @51 :Legacy.LiveLocationData; + liveLocationKalmanLegacyDEPRECATED @51 :Legacy.LiveLocationData; orbslamCorrectionDEPRECATED @45 :Legacy.OrbslamCorrection; liveUIDEPRECATED @14 :Legacy.LiveUI; sensorEventDEPRECATED @4 :SensorEventData; @@ -2365,5 +2610,9 @@ struct Event { sensorEventsDEPRECATED @11 :List(SensorEventData); lateralPlanDEPRECATED @64 :LateralPlan; navModelDEPRECATED @104 :NavModelData; + uiPlanDEPRECATED @106 :UiPlan; + liveLocationKalmanDEPRECATED @72 :LiveLocationKalman; + liveTracksDEPRECATED @16 :List(LiveTracksDEPRECATED); + onroadEventsDEPRECATED @68: List(Car.OnroadEventDEPRECATED); } } diff --git a/cereal/maptile.capnp b/cereal/maptile.capnp deleted file mode 100644 index c8a23a1822..0000000000 --- a/cereal/maptile.capnp +++ /dev/null @@ -1,49 +0,0 @@ -using Cxx = import "./include/c++.capnp"; -$Cxx.namespace("cereal"); - -@0xa086df597ef5d7a0; - -# Geometry -struct Point { - x @0: Float64; - y @1: Float64; - z @2: Float64; -} - -struct PolyLine { - points @0: List(Point); -} - -# Map features -struct Lane { - id @0 :Text; - - leftBoundary @1 :LaneBoundary; - rightBoundary @2 :LaneBoundary; - - leftAdjacentId @3 :Text; - rightAdjacentId @4 :Text; - - inboundIds @5 :List(Text); - outboundIds @6 :List(Text); - - struct LaneBoundary { - polyLine @0 :PolyLine; - startHeading @1 :Float32; # WRT north - } -} - -# Map tiles -struct TileSummary { - version @0 :Text; - updatedAt @1 :UInt64; # Millis since epoch - - level @2 :UInt8; - x @3 :UInt16; - y @4 :UInt16; -} - -struct MapTile { - summary @0 :TileSummary; - lanes @1 :List(Lane); -} diff --git a/cereal/messaging/.gitignore b/cereal/messaging/.gitignore deleted file mode 100644 index dbbe8e22ae..0000000000 --- a/cereal/messaging/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -demo -bridge -test_runner -*.o -*.os -*.d -*.a -*.so -messaging_pyx.cpp -build/ diff --git a/cereal/messaging/__init__.py b/cereal/messaging/__init__.py index d8115d1818..2466f6e9c0 100644 --- a/cereal/messaging/__init__.py +++ b/cereal/messaging/__init__.py @@ -1,7 +1,9 @@ # must be built with scons -from msgq.messaging.messaging_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \ +from msgq.ipc_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \ set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event -from msgq.messaging.messaging_pyx import MultiplePublishersError, MessagingError +from msgq.ipc_pyx import MultiplePublishersError, IpcError +from msgq import fake_event_handle, pub_sock, sub_sock, drain_sock_raw +import msgq import os import capnp @@ -13,30 +15,15 @@ from collections import deque from cereal import log from cereal.services import SERVICE_LIST -assert MultiplePublishersError -assert MessagingError -assert toggle_fake_events -assert set_fake_prefix -assert get_fake_prefix -assert delete_fake_prefix -assert wait_for_one_event - NO_TRAVERSAL_LIMIT = 2**64-1 -context = Context() - - -def fake_event_handle(endpoint: str, identifier: Optional[str] = None, override: bool = True, enable: bool = False) -> SocketEventHandle: - identifier = identifier or get_fake_prefix() - handle = SocketEventHandle(endpoint, identifier, override) - if override: - handle.enabled = enable - return handle +def reset_context(): + msgq.context = Context() -def log_from_bytes(dat: bytes) -> capnp.lib.capnp._DynamicStructReader: - with log.Event.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT) as msg: +def log_from_bytes(dat: bytes, struct: capnp.lib.capnp._StructModule = log.Event) -> capnp.lib.capnp._DynamicStructReader: + with struct.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT) as msg: return msg @@ -55,42 +42,6 @@ def new_message(service: Optional[str], size: Optional[int] = None, **kwargs) -> return dat -def pub_sock(endpoint: str) -> PubSocket: - sock = PubSocket() - sock.connect(context, endpoint) - return sock - - -def sub_sock(endpoint: str, poller: Optional[Poller] = None, addr: str = "127.0.0.1", - conflate: bool = False, timeout: Optional[int] = None) -> SubSocket: - sock = SubSocket() - sock.connect(context, endpoint, addr.encode('utf8'), conflate) - - if timeout is not None: - sock.setTimeout(timeout) - - if poller is not None: - poller.registerSocket(sock) - return sock - - -def drain_sock_raw(sock: SubSocket, wait_for_one: bool = False) -> List[bytes]: - """Receive all message currently available on the queue""" - ret: List[bytes] = [] - while 1: - if wait_for_one and len(ret) == 0: - dat = sock.receive() - else: - dat = sock.receive(non_blocking=True) - - if dat is None: - break - - ret.append(dat) - - return ret - - def drain_sock(sock: SubSocket, wait_for_one: bool = False) -> List[capnp.lib.capnp._DynamicStructReader]: """Receive all message currently available on the queue""" msgs = drain_sock_raw(sock, wait_for_one=wait_for_one) @@ -141,26 +92,76 @@ def recv_one_retry(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader: return log_from_bytes(dat) +class FrequencyTracker: + def __init__(self, service_freq: float, update_freq: float, is_poll: bool): + freq = max(min(service_freq, update_freq), 1.) + if is_poll: + min_freq = max_freq = freq + else: + max_freq = min(freq, update_freq) + if service_freq >= 2 * update_freq: + min_freq = update_freq + elif update_freq >= 2* service_freq: + min_freq = freq + else: + min_freq = min(freq, freq / 2.) + + self.min_freq = min_freq * 0.8 + self.max_freq = max_freq * 1.2 + self.recv_dts: Deque[float] = deque(maxlen=int(10 * freq)) + self.recv_dts_sum = 0.0 + self.recent_recv_dts: Deque[float] = deque(maxlen=int(freq)) + self.recent_recv_dts_sum = 0.0 + self.prev_time = 0.0 + + def record_recv_time(self, cur_time: float) -> None: + # TODO: Handle case where cur_time is less than prev_time + if self.prev_time > 1e-5: + dt = cur_time - self.prev_time + + if len(self.recv_dts) == self.recv_dts.maxlen: + self.recv_dts_sum -= self.recv_dts[0] + self.recv_dts.append(dt) + self.recv_dts_sum += dt + + if len(self.recent_recv_dts) == self.recent_recv_dts.maxlen: + self.recent_recv_dts_sum -= self.recent_recv_dts[0] + self.recent_recv_dts.append(dt) + self.recent_recv_dts_sum += dt + + self.prev_time = cur_time + + @property + def valid(self) -> bool: + if not self.recv_dts: + return False + + avg_freq = len(self.recv_dts) / self.recv_dts_sum + if self.min_freq <= avg_freq <= self.max_freq: + return True + + avg_freq_recent = len(self.recent_recv_dts) / self.recent_recv_dts_sum + return self.min_freq <= avg_freq_recent <= self.max_freq + + class SubMaster: def __init__(self, services: List[str], poll: Optional[str] = None, ignore_alive: Optional[List[str]] = None, ignore_avg_freq: Optional[List[str]] = None, ignore_valid: Optional[List[str]] = None, addr: str = "127.0.0.1", frequency: Optional[float] = None): self.frame = -1 + self.services = services self.seen = {s: False for s in services} self.updated = {s: False for s in services} self.recv_time = {s: 0. for s in services} self.recv_frame = {s: 0 for s in services} self.alive = {s: False for s in services} self.freq_ok = {s: False for s in services} - self.recv_dts: Dict[str, Deque[float]] = {} self.sock = {} self.data = {} self.valid = {} self.logMonoTime = {} - self.max_freq = {} - self.min_freq = {} - + self.freq_tracker: Dict[str, FrequencyTracker] = {} self.poller = Poller() polled_services = set([poll, ] if poll is not None else services) self.non_polled_services = set(services) - polled_services @@ -186,23 +187,8 @@ class SubMaster: self.data[s] = getattr(data.as_reader(), s) self.logMonoTime[s] = 0 - self.valid[s] = True # FIXME: this should default to False - - freq = max(min([SERVICE_LIST[s].frequency, self.update_freq]), 1.) - if s == poll: - max_freq = freq - min_freq = freq - else: - max_freq = min(freq, self.update_freq) - if SERVICE_LIST[s].frequency >= 2*self.update_freq: - min_freq = self.update_freq - elif self.update_freq >= 2*SERVICE_LIST[s].frequency: - min_freq = freq - else: - min_freq = min(freq, freq / 2.) - self.max_freq[s] = max_freq*1.2 - self.min_freq[s] = min_freq*0.8 - self.recv_dts[s] = deque(maxlen=int(10*freq)) + self.valid[s] = False + self.freq_tracker[s] = FrequencyTracker(SERVICE_LIST[s].frequency, self.update_freq, s == poll) def __getitem__(self, s: str) -> capnp.lib.capnp._DynamicStructReader: return self.data[s] @@ -222,7 +208,7 @@ class SubMaster: def update_msgs(self, cur_time: float, msgs: List[capnp.lib.capnp._DynamicStructReader]) -> None: self.frame += 1 - self.updated = dict.fromkeys(self.updated, False) + self.updated = dict.fromkeys(self.services, False) for msg in msgs: if msg is None: continue @@ -231,54 +217,30 @@ class SubMaster: self.seen[s] = True self.updated[s] = True - if self.recv_time[s] > 1e-5: - self.recv_dts[s].append(cur_time - self.recv_time[s]) + self.freq_tracker[s].record_recv_time(cur_time) self.recv_time[s] = cur_time self.recv_frame[s] = self.frame self.data[s] = getattr(msg, s) self.logMonoTime[s] = msg.logMonoTime self.valid[s] = msg.valid - for s in self.data: + for s in self.services: if SERVICE_LIST[s].frequency > 1e-5 and not self.simulation: # alive if delay is within 10x the expected frequency self.alive[s] = (cur_time - self.recv_time[s]) < (10. / SERVICE_LIST[s].frequency) - - # check average frequency; slow to fall, quick to recover - dts = self.recv_dts[s] - assert dts.maxlen is not None - recent_dts = list(dts)[-int(dts.maxlen / 10):] - try: - avg_freq = 1 / (sum(dts) / len(dts)) - avg_freq_recent = 1 / (sum(recent_dts) / len(recent_dts)) - except ZeroDivisionError: - avg_freq = 0 - avg_freq_recent = 0 - - avg_freq_ok = self.min_freq[s] <= avg_freq <= self.max_freq[s] - recent_freq_ok = self.min_freq[s] <= avg_freq_recent <= self.max_freq[s] - self.freq_ok[s] = avg_freq_ok or recent_freq_ok + self.freq_ok[s] = self.freq_tracker[s].valid else: self.freq_ok[s] = True - if self.simulation: - self.alive[s] = self.seen[s] # alive is defined as seen when simulation flag set - else: - self.alive[s] = True + self.alive[s] = self.seen[s] if self.simulation else True def all_alive(self, service_list: Optional[List[str]] = None) -> bool: - if service_list is None: - service_list = list(self.sock.keys()) - return all(self.alive[s] for s in service_list if s not in self.ignore_alive) + return all(self.alive[s] for s in (service_list or self.services) if s not in self.ignore_alive) def all_freq_ok(self, service_list: Optional[List[str]] = None) -> bool: - if service_list is None: - service_list = list(self.sock.keys()) - return all(self.freq_ok[s] for s in service_list if self._check_avg_freq(s)) + return all(self.freq_ok[s] for s in (service_list or self.services) if self._check_avg_freq(s)) def all_valid(self, service_list: Optional[List[str]] = None) -> bool: - if service_list is None: - service_list = list(self.sock.keys()) - return all(self.valid[s] for s in service_list if s not in self.ignore_valid) + return all(self.valid[s] for s in (service_list or self.services) if s not in self.ignore_valid) def all_checks(self, service_list: Optional[List[str]] = None) -> bool: return self.all_alive(service_list) and self.all_freq_ok(service_list) and self.all_valid(service_list) diff --git a/cereal/messaging/bridge.cc b/cereal/messaging/bridge.cc index b16548314b..93af988358 100644 --- a/cereal/messaging/bridge.cc +++ b/cereal/messaging/bridge.cc @@ -1,25 +1,10 @@ -#include #include -#include -#include -#include -#include - -typedef void (*sighandler_t)(int sig); +#include "cereal/messaging/msgq_to_zmq.h" #include "cereal/services.h" -#include "msgq/messaging/impl_msgq.h" -#include "msgq/messaging/impl_zmq.h" +#include "common/util.h" -std::atomic do_exit = false; -static void set_do_exit(int sig) { - do_exit = true; -} - -void sigpipe_handler(int sig) { - assert(sig == SIGPIPE); - std::cout << "SIGPIPE received" << std::endl; -} +ExitHandler do_exit; static std::vector get_services(std::string whitelist_str, bool zmq_to_msgq) { std::vector service_list; @@ -34,41 +19,22 @@ static std::vector get_services(std::string whitelist_str, bool zmq return service_list; } -int main(int argc, char** argv) { - signal(SIGPIPE, (sighandler_t)sigpipe_handler); - signal(SIGINT, (sighandler_t)set_do_exit); - signal(SIGTERM, (sighandler_t)set_do_exit); - - bool zmq_to_msgq = argc > 2; - std::string ip = zmq_to_msgq ? argv[1] : "127.0.0.1"; - std::string whitelist_str = zmq_to_msgq ? std::string(argv[2]) : ""; +void msgq_to_zmq(const std::vector &endpoints, const std::string &ip) { + MsgqToZmq bridge; + bridge.run(endpoints, ip); +} - Poller *poller; - Context *pub_context; - Context *sub_context; - if (zmq_to_msgq) { // republishes zmq debugging messages as msgq - poller = new ZMQPoller(); - pub_context = new MSGQContext(); - sub_context = new ZMQContext(); - } else { - poller = new MSGQPoller(); - pub_context = new ZMQContext(); - sub_context = new MSGQContext(); - } +void zmq_to_msgq(const std::vector &endpoints, const std::string &ip) { + auto poller = std::make_unique(); + auto pub_context = std::make_unique(); + auto sub_context = std::make_unique(); + std::map sub2pub; - std::map sub2pub; - for (auto endpoint : get_services(whitelist_str, zmq_to_msgq)) { - PubSocket * pub_sock; - SubSocket * sub_sock; - if (zmq_to_msgq) { - pub_sock = new MSGQPubSocket(); - sub_sock = new ZMQSubSocket(); - } else { - pub_sock = new ZMQPubSocket(); - sub_sock = new MSGQSubSocket(); - } - pub_sock->connect(pub_context, endpoint); - sub_sock->connect(sub_context, endpoint, ip, false); + for (auto endpoint : endpoints) { + auto pub_sock = new MSGQPubSocket(); + auto sub_sock = new ZMQSubSocket(); + pub_sock->connect(pub_context.get(), endpoint); + sub_sock->connect(sub_context.get(), endpoint, ip, false); poller->registerSocket(sub_sock); sub2pub[sub_sock] = pub_sock; @@ -76,17 +42,30 @@ int main(int argc, char** argv) { while (!do_exit) { for (auto sub_sock : poller->poll(100)) { - Message * msg = sub_sock->receive(); - if (msg == NULL) continue; - int ret; - do { - ret = sub2pub[sub_sock]->sendMessage(msg); - } while (ret == -1 && errno == EINTR && !do_exit); - assert(ret >= 0 || do_exit); - delete msg; - - if (do_exit) break; + std::unique_ptr msg(sub_sock->receive(true)); + if (msg) { + sub2pub[sub_sock]->sendMessage(msg.get()); + } } } + + // Clean up allocated sockets + for (auto &[sub_sock, pub_sock] : sub2pub) { + delete sub_sock; + delete pub_sock; + } +} + +int main(int argc, char **argv) { + bool is_zmq_to_msgq = argc > 2; + std::string ip = is_zmq_to_msgq ? argv[1] : "127.0.0.1"; + std::string whitelist_str = is_zmq_to_msgq ? std::string(argv[2]) : ""; + std::vector endpoints = get_services(whitelist_str, is_zmq_to_msgq); + + if (is_zmq_to_msgq) { + zmq_to_msgq(endpoints, ip); + } else { + msgq_to_zmq(endpoints, ip); + } return 0; } diff --git a/cereal/messaging/messaging.h b/cereal/messaging/messaging.h index b0f8b55355..fb9c261f2b 100644 --- a/cereal/messaging/messaging.h +++ b/cereal/messaging/messaging.h @@ -5,19 +5,12 @@ #include #include #include -#include #include #include "cereal/gen/cpp/log.capnp.h" -#include "msgq/messaging/messaging.h" - -#ifdef __APPLE__ -#define CLOCK_BOOTTIME CLOCK_MONOTONIC -#endif - -#define MSG_MULTIPLE_PUBLISHERS 100 - +#include "common/timing.h" +#include "msgq/ipc.h" class SubMaster { public: @@ -53,10 +46,7 @@ public: cereal::Event::Builder initEvent(bool valid = true) { cereal::Event::Builder event = initRoot(); - struct timespec t; - clock_gettime(CLOCK_BOOTTIME, &t); - uint64_t current_time = t.tv_sec * 1000000000ULL + t.tv_nsec; - event.setLogMonoTime(current_time); + event.setLogMonoTime(nanos_since_boot()); event.setValid(valid); return event; } diff --git a/cereal/messaging/msgq_to_zmq.cc b/cereal/messaging/msgq_to_zmq.cc new file mode 100644 index 0000000000..ce626f2aad --- /dev/null +++ b/cereal/messaging/msgq_to_zmq.cc @@ -0,0 +1,144 @@ +#include "cereal/messaging/msgq_to_zmq.h" + +#include + +#include "common/util.h" + +extern ExitHandler do_exit; + +// Max messages to process per socket per poll +constexpr int MAX_MESSAGES_PER_SOCKET = 50; + +static std::string recv_zmq_msg(void *sock) { + zmq_msg_t msg; + zmq_msg_init(&msg); + std::string ret; + if (zmq_msg_recv(&msg, sock, 0) > 0) { + ret.assign((char *)zmq_msg_data(&msg), zmq_msg_size(&msg)); + } + zmq_msg_close(&msg); + return ret; +} + +void MsgqToZmq::run(const std::vector &endpoints, const std::string &ip) { + zmq_context = std::make_unique(); + msgq_context = std::make_unique(); + + // Create ZMQPubSockets for each endpoint + for (const auto &endpoint : endpoints) { + auto &socket_pair = socket_pairs.emplace_back(); + socket_pair.endpoint = endpoint; + socket_pair.pub_sock = std::make_unique(); + int ret = socket_pair.pub_sock->connect(zmq_context.get(), endpoint); + if (ret != 0) { + printf("Failed to create ZMQ publisher for [%s]: %s\n", endpoint.c_str(), zmq_strerror(zmq_errno())); + return; + } + } + + // Start ZMQ monitoring thread to monitor socket events + std::thread thread(&MsgqToZmq::zmqMonitorThread, this); + + // Main loop for processing messages + while (!do_exit) { + { + std::unique_lock lk(mutex); + cv.wait(lk, [this]() { return do_exit || !sub2pub.empty(); }); + if (do_exit) break; + + for (auto sub_sock : msgq_poller->poll(100)) { + // Process messages for each socket + ZMQPubSocket *pub_sock = sub2pub.at(sub_sock); + for (int i = 0; i < MAX_MESSAGES_PER_SOCKET; ++i) { + auto msg = std::unique_ptr(sub_sock->receive(true)); + if (!msg) break; + + while (pub_sock->sendMessage(msg.get()) == -1) { + if (errno != EINTR) break; + } + } + } + } + util::sleep_for(1); // Give zmqMonitorThread a chance to acquire the mutex + } + + thread.join(); +} + +void MsgqToZmq::zmqMonitorThread() { + std::vector pollitems; + + // Set up ZMQ monitor for each pub socket + for (int i = 0; i < socket_pairs.size(); ++i) { + std::string addr = "inproc://op-bridge-monitor-" + std::to_string(i); + zmq_socket_monitor(socket_pairs[i].pub_sock->sock, addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED); + + void *monitor_socket = zmq_socket(zmq_context->getRawContext(), ZMQ_PAIR); + zmq_connect(monitor_socket, addr.c_str()); + pollitems.emplace_back(zmq_pollitem_t{.socket = monitor_socket, .events = ZMQ_POLLIN}); + } + + while (!do_exit) { + int ret = zmq_poll(pollitems.data(), pollitems.size(), 1000); + if (ret < 0) { + if (errno == EINTR) { + // Due to frequent EINTR signals from msgq, introduce a brief delay (200 ms) + // to reduce CPU usage during retry attempts. + util::sleep_for(200); + } + continue; + } + + for (int i = 0; i < pollitems.size(); ++i) { + if (pollitems[i].revents & ZMQ_POLLIN) { + // First frame in message contains event number and value + std::string frame = recv_zmq_msg(pollitems[i].socket); + if (frame.empty()) continue; + + uint16_t event_type = *(uint16_t *)(frame.data()); + + // Second frame in message contains event address + frame = recv_zmq_msg(pollitems[i].socket); + if (frame.empty()) continue; + + std::unique_lock lk(mutex); + auto &pair = socket_pairs[i]; + if (event_type & ZMQ_EVENT_ACCEPTED) { + printf("socket [%s] connected\n", pair.endpoint.c_str()); + if (++pair.connected_clients == 1) { + // Create new MSGQ subscriber socket and map to ZMQ publisher + pair.sub_sock = std::make_unique(); + pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1"); + sub2pub[pair.sub_sock.get()] = pair.pub_sock.get(); + registerSockets(); + } + } else if (event_type & ZMQ_EVENT_DISCONNECTED) { + printf("socket [%s] disconnected\n", pair.endpoint.c_str()); + if (pair.connected_clients == 0 || --pair.connected_clients == 0) { + // Remove MSGQ subscriber socket from mapping and reset it + sub2pub.erase(pair.sub_sock.get()); + pair.sub_sock.reset(nullptr); + registerSockets(); + } + } + cv.notify_one(); + } + } + } + + // Clean up monitor sockets + for (int i = 0; i < pollitems.size(); ++i) { + zmq_socket_monitor(socket_pairs[i].pub_sock->sock, nullptr, 0); + zmq_close(pollitems[i].socket); + } + cv.notify_one(); +} + +void MsgqToZmq::registerSockets() { + msgq_poller = std::make_unique(); + for (const auto &socket_pair : socket_pairs) { + if (socket_pair.sub_sock) { + msgq_poller->registerSocket(socket_pair.sub_sock.get()); + } + } +} diff --git a/cereal/messaging/msgq_to_zmq.h b/cereal/messaging/msgq_to_zmq.h new file mode 100644 index 0000000000..ebdbe5df69 --- /dev/null +++ b/cereal/messaging/msgq_to_zmq.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#define private public +#include "msgq/impl_msgq.h" +#include "msgq/impl_zmq.h" + +class MsgqToZmq { +public: + MsgqToZmq() {} + void run(const std::vector &endpoints, const std::string &ip); + +protected: + void registerSockets(); + void zmqMonitorThread(); + + struct SocketPair { + std::string endpoint; + std::unique_ptr pub_sock; + std::unique_ptr sub_sock; + int connected_clients = 0; + }; + + std::unique_ptr msgq_context; + std::unique_ptr zmq_context; + std::mutex mutex; + std::condition_variable cv; + std::unique_ptr msgq_poller; + std::map sub2pub; + std::vector socket_pairs; +}; diff --git a/cereal/messaging/socketmaster.cc b/cereal/messaging/socketmaster.cc index cd697b1f51..1a7a48980e 100644 --- a/cereal/messaging/socketmaster.cc +++ b/cereal/messaging/socketmaster.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -7,15 +6,8 @@ #include "cereal/services.h" #include "cereal/messaging/messaging.h" - const bool SIMULATION = (getenv("SIMULATION") != nullptr) && (std::string(getenv("SIMULATION")) == "1"); -static inline uint64_t nanos_since_boot() { - struct timespec t; - clock_gettime(CLOCK_BOOTTIME, &t); - return t.tv_sec * 1000000000ULL + t.tv_nsec; -} - static inline bool inList(const std::vector &list, const char *value) { for (auto &v : list) { if (strcmp(value, v) == 0) return true; @@ -42,7 +34,7 @@ struct SubMaster::SubMessage { std::string name; SubSocket *socket = nullptr; int freq = 0; - bool updated = false, alive = false, valid = true, ignore_alive; + bool updated = false, alive = false, valid = false, ignore_alive; uint64_t rcv_time = 0, rcv_frame = 0; void *allocated_msg_reader = nullptr; bool is_polled = false; diff --git a/cereal/messaging/tests/test_fake.py b/cereal/messaging/tests/test_fake.py deleted file mode 100644 index 1d3521745d..0000000000 --- a/cereal/messaging/tests/test_fake.py +++ /dev/null @@ -1,193 +0,0 @@ -import os -import unittest -import multiprocessing -import platform -from parameterized import parameterized_class -from typing import Optional - -import cereal.messaging as messaging - -WAIT_TIMEOUT = 5 - - -@unittest.skipIf(platform.system() == "Darwin", "Events not supported on macOS") -class TestEvents(unittest.TestCase): - - def test_mutation(self): - handle = messaging.fake_event_handle("carState") - event = handle.recv_called_event - - self.assertFalse(event.peek()) - event.set() - self.assertTrue(event.peek()) - event.clear() - self.assertFalse(event.peek()) - - del event - - def test_wait(self): - handle = messaging.fake_event_handle("carState") - event = handle.recv_called_event - - event.set() - try: - event.wait(WAIT_TIMEOUT) - self.assertTrue(event.peek()) - except RuntimeError: - self.fail("event.wait() timed out") - - def test_wait_multiprocess(self): - handle = messaging.fake_event_handle("carState") - event = handle.recv_called_event - - def set_event_run(): - event.set() - - try: - p = multiprocessing.Process(target=set_event_run) - p.start() - event.wait(WAIT_TIMEOUT) - self.assertTrue(event.peek()) - except RuntimeError: - self.fail("event.wait() timed out") - - p.kill() - - def test_wait_zero_timeout(self): - handle = messaging.fake_event_handle("carState") - event = handle.recv_called_event - - try: - event.wait(0) - self.fail("event.wait() did not time out") - except RuntimeError: - self.assertFalse(event.peek()) - - -@unittest.skipIf(platform.system() == "Darwin", "FakeSockets not supported on macOS") -@unittest.skipIf("ZMQ" in os.environ, "FakeSockets not supported on ZMQ") -@parameterized_class([{"prefix": None}, {"prefix": "test"}]) -class TestFakeSockets(unittest.TestCase): - prefix: Optional[str] = None - - def setUp(self): - messaging.toggle_fake_events(True) - if self.prefix is not None: - messaging.set_fake_prefix(self.prefix) - else: - messaging.delete_fake_prefix() - - def tearDown(self): - messaging.toggle_fake_events(False) - messaging.delete_fake_prefix() - - def test_event_handle_init(self): - handle = messaging.fake_event_handle("controlsState", override=True) - - self.assertFalse(handle.enabled) - self.assertGreaterEqual(handle.recv_called_event.fd, 0) - self.assertGreaterEqual(handle.recv_ready_event.fd, 0) - - def test_non_managed_socket_state(self): - # non managed socket should have zero state - _ = messaging.pub_sock("ubloxGnss") - - handle = messaging.fake_event_handle("ubloxGnss", override=False) - - self.assertFalse(handle.enabled) - self.assertEqual(handle.recv_called_event.fd, 0) - self.assertEqual(handle.recv_ready_event.fd, 0) - - def test_managed_socket_state(self): - # managed socket should not change anything about the state - handle = messaging.fake_event_handle("ubloxGnss") - handle.enabled = True - - expected_enabled = handle.enabled - expected_recv_called_fd = handle.recv_called_event.fd - expected_recv_ready_fd = handle.recv_ready_event.fd - - _ = messaging.pub_sock("ubloxGnss") - - self.assertEqual(handle.enabled, expected_enabled) - self.assertEqual(handle.recv_called_event.fd, expected_recv_called_fd) - self.assertEqual(handle.recv_ready_event.fd, expected_recv_ready_fd) - - def test_sockets_enable_disable(self): - carState_handle = messaging.fake_event_handle("ubloxGnss", enable=True) - recv_called = carState_handle.recv_called_event - recv_ready = carState_handle.recv_ready_event - - pub_sock = messaging.pub_sock("ubloxGnss") - sub_sock = messaging.sub_sock("ubloxGnss") - - try: - carState_handle.enabled = True - recv_ready.set() - pub_sock.send(b"test") - _ = sub_sock.receive() - self.assertTrue(recv_called.peek()) - recv_called.clear() - - carState_handle.enabled = False - recv_ready.set() - pub_sock.send(b"test") - _ = sub_sock.receive() - self.assertFalse(recv_called.peek()) - except RuntimeError: - self.fail("event.wait() timed out") - - def test_synced_pub_sub(self): - def daemon_repub_process_run(): - pub_sock = messaging.pub_sock("ubloxGnss") - sub_sock = messaging.sub_sock("carState") - - frame = -1 - while True: - frame += 1 - msg = sub_sock.receive(non_blocking=True) - if msg is None: - print("none received") - continue - - bts = frame.to_bytes(8, 'little') - pub_sock.send(bts) - - carState_handle = messaging.fake_event_handle("carState", enable=True) - recv_called = carState_handle.recv_called_event - recv_ready = carState_handle.recv_ready_event - - p = multiprocessing.Process(target=daemon_repub_process_run) - p.start() - - pub_sock = messaging.pub_sock("carState") - sub_sock = messaging.sub_sock("ubloxGnss") - - try: - for i in range(10): - recv_called.wait(WAIT_TIMEOUT) - recv_called.clear() - - if i == 0: - sub_sock.receive(non_blocking=True) - - bts = i.to_bytes(8, 'little') - pub_sock.send(bts) - - recv_ready.set() - recv_called.wait(WAIT_TIMEOUT) - - msg = sub_sock.receive(non_blocking=True) - self.assertIsNotNone(msg) - self.assertEqual(len(msg), 8) - - frame = int.from_bytes(msg, 'little') - self.assertEqual(frame, i) - except RuntimeError: - self.fail("event.wait() timed out") - finally: - p.kill() - - -if __name__ == "__main__": - unittest.main() diff --git a/cereal/messaging/tests/test_messaging.py b/cereal/messaging/tests/test_messaging.py old mode 100755 new mode 100644 index 381cec03ff..6234a11c08 --- a/cereal/messaging/tests/test_messaging.py +++ b/cereal/messaging/tests/test_messaging.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 import os import capnp import multiprocessing @@ -6,8 +5,8 @@ import numbers import random import threading import time -import unittest from parameterized import parameterized +import pytest from cereal import log, car import cereal.messaging as messaging @@ -28,11 +27,6 @@ def zmq_sleep(t=1): if "ZMQ" in os.environ: time.sleep(t) -def zmq_expected_failure(func): - if "ZMQ" in os.environ: - return unittest.expectedFailure(func) - else: - return func # TODO: this should take any capnp struct and returrn a msg with random populated data def random_carstate(): @@ -56,66 +50,13 @@ def delayed_send(delay, sock, dat): sock.send(dat) threading.Timer(delay, send_func).start() -class TestPubSubSockets(unittest.TestCase): - - def setUp(self): - # ZMQ pub socket takes too long to die - # sleep to prevent multiple publishers error between tests - zmq_sleep() - - def test_pub_sub(self): - sock = random_sock() - pub_sock = messaging.pub_sock(sock) - sub_sock = messaging.sub_sock(sock, conflate=False, timeout=None) - zmq_sleep(3) - - for _ in range(1000): - msg = random_bytes() - pub_sock.send(msg) - recvd = sub_sock.receive() - self.assertEqual(msg, recvd) - - def test_conflate(self): - sock = random_sock() - pub_sock = messaging.pub_sock(sock) - for conflate in [True, False]: - for _ in range(10): - num_msgs = random.randint(3, 10) - sub_sock = messaging.sub_sock(sock, conflate=conflate, timeout=None) - zmq_sleep() - - sent_msgs = [] - for __ in range(num_msgs): - msg = random_bytes() - pub_sock.send(msg) - sent_msgs.append(msg) - time.sleep(0.1) - recvd_msgs = messaging.drain_sock_raw(sub_sock) - if conflate: - self.assertEqual(len(recvd_msgs), 1) - else: - # TODO: compare actual data - self.assertEqual(len(recvd_msgs), len(sent_msgs)) - - def test_receive_timeout(self): - sock = random_sock() - for _ in range(10): - timeout = random.randrange(200) - sub_sock = messaging.sub_sock(sock, timeout=timeout) - zmq_sleep() - - start_time = time.monotonic() - recvd = sub_sock.receive() - self.assertLess(time.monotonic() - start_time, 0.2) - assert recvd is None - -class TestMessaging(unittest.TestCase): +class TestMessaging: def setUp(self): # TODO: ZMQ tests are too slow; all sleeps will need to be # replaced with logic to block on the necessary condition if "ZMQ" in os.environ: - raise unittest.SkipTest + pytest.skip() # ZMQ pub socket takes too long to die # sleep to prevent multiple publishers error between tests @@ -127,9 +68,9 @@ class TestMessaging(unittest.TestCase): msg = messaging.new_message(evt) except capnp.lib.capnp.KjException: msg = messaging.new_message(evt, random.randrange(200)) - self.assertLess(time.monotonic() - msg.logMonoTime, 0.1) - self.assertFalse(msg.valid) - self.assertEqual(evt, msg.which()) + assert (time.monotonic() - msg.logMonoTime) < 0.1 + assert not msg.valid + assert evt == msg.which() @parameterized.expand(events) def test_pub_sock(self, evt): @@ -151,8 +92,8 @@ class TestMessaging(unittest.TestCase): # no wait and no msgs in queue msgs = func(sub_sock) - self.assertIsInstance(msgs, list) - self.assertEqual(len(msgs), 0) + assert isinstance(msgs, list) + assert len(msgs) == 0 # no wait but msgs are queued up num_msgs = random.randrange(3, 10) @@ -160,9 +101,9 @@ class TestMessaging(unittest.TestCase): pub_sock.send(messaging.new_message(sock).to_bytes()) time.sleep(0.1) msgs = func(sub_sock) - self.assertIsInstance(msgs, list) - self.assertTrue(all(isinstance(msg, expected_type) for msg in msgs)) - self.assertEqual(len(msgs), num_msgs) + assert isinstance(msgs, list) + assert all(isinstance(msg, expected_type) for msg in msgs) + assert len(msgs) == num_msgs def test_recv_sock(self): sock = "carState" @@ -172,14 +113,14 @@ class TestMessaging(unittest.TestCase): # no wait and no msg in queue, socket should timeout recvd = messaging.recv_sock(sub_sock) - self.assertTrue(recvd is None) + assert recvd is None # no wait and one msg in queue msg = random_carstate() pub_sock.send(msg.to_bytes()) time.sleep(0.01) recvd = messaging.recv_sock(sub_sock) - self.assertIsInstance(recvd, capnp._DynamicStructReader) + assert isinstance(recvd, capnp._DynamicStructReader) # https://github.com/python/mypy/issues/13038 assert_carstate(msg.carState, recvd.carState) @@ -191,16 +132,16 @@ class TestMessaging(unittest.TestCase): # no msg in queue, socket should timeout recvd = messaging.recv_one(sub_sock) - self.assertTrue(recvd is None) + assert recvd is None # one msg in queue msg = random_carstate() pub_sock.send(msg.to_bytes()) recvd = messaging.recv_one(sub_sock) - self.assertIsInstance(recvd, capnp._DynamicStructReader) + assert isinstance(recvd, capnp._DynamicStructReader) assert_carstate(msg.carState, recvd.carState) - @zmq_expected_failure + @pytest.mark.xfail(condition="ZMQ" in os.environ, reason='ZMQ detected') def test_recv_one_or_none(self): sock = "carState" pub_sock = messaging.pub_sock(sock) @@ -209,13 +150,13 @@ class TestMessaging(unittest.TestCase): # no msg in queue, socket shouldn't block recvd = messaging.recv_one_or_none(sub_sock) - self.assertTrue(recvd is None) + assert recvd is None # one msg in queue msg = random_carstate() pub_sock.send(msg.to_bytes()) recvd = messaging.recv_one_or_none(sub_sock) - self.assertIsInstance(recvd, capnp._DynamicStructReader) + assert isinstance(recvd, capnp._DynamicStructReader) assert_carstate(msg.carState, recvd.carState) def test_recv_one_retry(self): @@ -227,21 +168,18 @@ class TestMessaging(unittest.TestCase): # this test doesn't work with ZMQ since multiprocessing interrupts it if "ZMQ" not in os.environ: - # wait 15 socket timeouts and make sure it's still retrying + # wait 5 socket timeouts and make sure it's still retrying p = multiprocessing.Process(target=messaging.recv_one_retry, args=(sub_sock,)) p.start() - time.sleep(sock_timeout*15) - self.assertTrue(p.is_alive()) + time.sleep(sock_timeout*5) + assert p.is_alive() p.terminate() - # wait 15 socket timeouts before sending + # wait 5 socket timeouts before sending msg = random_carstate() - delayed_send(sock_timeout*15, pub_sock, msg.to_bytes()) + delayed_send(sock_timeout*5, pub_sock, msg.to_bytes()) start_time = time.monotonic() recvd = messaging.recv_one_retry(sub_sock) - self.assertGreaterEqual(time.monotonic() - start_time, sock_timeout*15) - self.assertIsInstance(recvd, capnp._DynamicStructReader) + assert (time.monotonic() - start_time) >= sock_timeout*5 + assert isinstance(recvd, capnp._DynamicStructReader) assert_carstate(msg.carState, recvd.carState) - -if __name__ == "__main__": - unittest.main() diff --git a/cereal/messaging/tests/test_poller.py b/cereal/messaging/tests/test_poller.py deleted file mode 100644 index bcff5e40ca..0000000000 --- a/cereal/messaging/tests/test_poller.py +++ /dev/null @@ -1,142 +0,0 @@ -import unittest -import time -import cereal.messaging as messaging - -import concurrent.futures - - -def poller(): - context = messaging.Context() - - p = messaging.Poller() - - sub = messaging.SubSocket() - sub.connect(context, 'controlsState') - p.registerSocket(sub) - - socks = p.poll(10000) - r = [s.receive(non_blocking=True) for s in socks] - - return r - - -class TestPoller(unittest.TestCase): - def test_poll_once(self): - context = messaging.Context() - - pub = messaging.PubSocket() - pub.connect(context, 'controlsState') - - with concurrent.futures.ThreadPoolExecutor() as e: - poll = e.submit(poller) - - time.sleep(0.1) # Slow joiner syndrome - - # Send message - pub.send(b"a") - - # Wait for poll result - result = poll.result() - - del pub - context.term() - - self.assertEqual(result, [b"a"]) - - def test_poll_and_create_many_subscribers(self): - context = messaging.Context() - - pub = messaging.PubSocket() - pub.connect(context, 'controlsState') - - with concurrent.futures.ThreadPoolExecutor() as e: - poll = e.submit(poller) - - time.sleep(0.1) # Slow joiner syndrome - c = messaging.Context() - for _ in range(10): - messaging.SubSocket().connect(c, 'controlsState') - - time.sleep(0.1) - - # Send message - pub.send(b"a") - - # Wait for poll result - result = poll.result() - - del pub - context.term() - - self.assertEqual(result, [b"a"]) - - def test_multiple_publishers_exception(self): - context = messaging.Context() - - with self.assertRaises(messaging.MultiplePublishersError): - pub1 = messaging.PubSocket() - pub1.connect(context, 'controlsState') - - pub2 = messaging.PubSocket() - pub2.connect(context, 'controlsState') - - pub1.send(b"a") - - del pub1 - del pub2 - context.term() - - def test_multiple_messages(self): - context = messaging.Context() - - pub = messaging.PubSocket() - pub.connect(context, 'controlsState') - - sub = messaging.SubSocket() - sub.connect(context, 'controlsState') - - time.sleep(0.1) # Slow joiner - - for i in range(1, 100): - pub.send(b'a'*i) - - msg_seen = False - i = 1 - while True: - r = sub.receive(non_blocking=True) - - if r is not None: - self.assertEqual(b'a'*i, r) - - msg_seen = True - i += 1 - - if r is None and msg_seen: # ZMQ sometimes receives nothing on the first receive - break - - del pub - del sub - context.term() - - def test_conflate(self): - context = messaging.Context() - - pub = messaging.PubSocket() - pub.connect(context, 'controlsState') - - sub = messaging.SubSocket() - sub.connect(context, 'controlsState', conflate=True) - - time.sleep(0.1) # Slow joiner - pub.send(b'a') - pub.send(b'b') - - self.assertEqual(b'b', sub.receive()) - - del pub - del sub - context.term() - - -if __name__ == "__main__": - unittest.main() diff --git a/cereal/messaging/tests/test_pub_sub_master.py b/cereal/messaging/tests/test_pub_sub_master.py old mode 100755 new mode 100644 index 81a1cf2d57..e9bc7a85cb --- a/cereal/messaging/tests/test_pub_sub_master.py +++ b/cereal/messaging/tests/test_pub_sub_master.py @@ -1,8 +1,6 @@ -#!/usr/bin/env python3 import random import time from typing import Sized, cast -import unittest import cereal.messaging as messaging from cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \ @@ -10,9 +8,9 @@ from cereal.messaging.tests.test_messaging import events, random_sock, random_so zmq_sleep -class TestSubMaster(unittest.TestCase): +class TestSubMaster: - def setUp(self): + def setup_method(self): # ZMQ pub socket takes too long to die # sleep to prevent multiple publishers error between tests zmq_sleep(3) @@ -21,21 +19,21 @@ class TestSubMaster(unittest.TestCase): sm = messaging.SubMaster(events) for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive, sm.sock, sm.data, sm.logMonoTime, sm.valid]: - self.assertEqual(len(cast(Sized, p)), len(events)) + assert len(cast(Sized, p)) == len(events) def test_init_state(self): socks = random_socks() sm = messaging.SubMaster(socks) - self.assertEqual(sm.frame, -1) - self.assertFalse(any(sm.updated.values())) - self.assertFalse(any(sm.alive.values())) - self.assertTrue(all(t == 0. for t in sm.recv_time.values())) - self.assertTrue(all(f == 0 for f in sm.recv_frame.values())) - self.assertTrue(all(t == 0 for t in sm.logMonoTime.values())) + assert sm.frame == -1 + assert not any(sm.updated.values()) + assert not any(sm.alive.values()) + assert all(t == 0. for t in sm.recv_time.values()) + assert all(f == 0 for f in sm.recv_frame.values()) + assert all(t == 0 for t in sm.logMonoTime.values()) for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive, sm.sock, sm.data, sm.logMonoTime, sm.valid]: - self.assertEqual(len(cast(Sized, p)), len(socks)) + assert len(cast(Sized, p)) == len(socks) def test_getitem(self): sock = "carState" @@ -59,20 +57,19 @@ class TestSubMaster(unittest.TestCase): msg = messaging.new_message(sock) pub_sock.send(msg.to_bytes()) sm.update(1000) - self.assertEqual(sm.frame, i) - self.assertTrue(all(sm.updated.values())) + assert sm.frame == i + assert all(sm.updated.values()) def test_update_timeout(self): sock = random_sock() sm = messaging.SubMaster([sock,]) - for _ in range(5): - timeout = random.randrange(1000, 5000) - start_time = time.monotonic() - sm.update(timeout) - t = time.monotonic() - start_time - self.assertGreaterEqual(t, timeout/1000.) - self.assertLess(t, 5) - self.assertFalse(any(sm.updated.values())) + timeout = random.randrange(1000, 3000) + start_time = time.monotonic() + sm.update(timeout) + t = time.monotonic() - start_time + assert t >= timeout/1000. + assert t < 3 + assert not any(sm.updated.values()) def test_avg_frequency_checks(self): for poll in (True, False): @@ -91,8 +88,8 @@ class TestSubMaster(unittest.TestCase): for service, (max_freq, min_freq) in checks.items(): if max_freq is not None: assert sm._check_avg_freq(service) - assert sm.max_freq[service] == max_freq*1.2 - assert sm.min_freq[service] == min_freq*0.8 + assert sm.freq_tracker[service].max_freq == max_freq*1.2 + assert sm.freq_tracker[service].min_freq == min_freq*0.8 else: assert not sm._check_avg_freq(service) @@ -118,12 +115,12 @@ class TestSubMaster(unittest.TestCase): pub_sock.send(msg.to_bytes()) time.sleep(0.01) sm.update(1000) - self.assertEqual(sm[sock].vEgo, n) + assert sm[sock].vEgo == n -class TestPubMaster(unittest.TestCase): +class TestPubMaster: - def setUp(self): + def setup_method(self): # ZMQ pub socket takes too long to die # sleep to prevent multiple publishers error between tests zmq_sleep(3) @@ -156,8 +153,4 @@ class TestPubMaster(unittest.TestCase): if capnp: msg.clear_write_flag() msg = msg.to_bytes() - self.assertEqual(msg, recvd, i) - - -if __name__ == "__main__": - unittest.main() + assert msg == recvd, i diff --git a/cereal/messaging/tests/test_services.py b/cereal/messaging/tests/test_services.py old mode 100755 new mode 100644 index 0fac379e43..8bfd2ea978 --- a/cereal/messaging/tests/test_services.py +++ b/cereal/messaging/tests/test_services.py @@ -1,25 +1,21 @@ -#!/usr/bin/env python3 import os import tempfile from typing import Dict -import unittest from parameterized import parameterized import cereal.services as services from cereal.services import SERVICE_LIST -class TestServices(unittest.TestCase): +class TestServices: @parameterized.expand(SERVICE_LIST.keys()) def test_services(self, s): service = SERVICE_LIST[s] - self.assertTrue(service.frequency <= 104) + assert service.frequency <= 104 + assert service.decimation != 0 def test_generated_header(self): with tempfile.NamedTemporaryFile(suffix=".h") as f: - ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name}") - self.assertEqual(ret, 0, "generated services header is not valid C") - -if __name__ == "__main__": - unittest.main() + ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name} -std=c++11") + assert ret == 0, "generated services header is not valid C" diff --git a/cereal/services.py b/cereal/services.py index 2ab28f6d52..87fdca77b7 100755 --- a/cereal/services.py +++ b/cereal/services.py @@ -16,14 +16,16 @@ _services: dict[str, tuple] = { "gyroscope2": (True, 100., 100), "accelerometer": (True, 104., 104), "accelerometer2": (True, 100., 100), - "magnetometer": (True, 25., 25), + "magnetometer": (True, 25.), "lightSensor": (True, 100., 100), "temperatureSensor": (True, 2., 200), "temperatureSensor2": (True, 2., 200), "gpsNMEA": (True, 9.), "deviceState": (True, 2., 1), - "can": (True, 100., 1223), # decimation gives ~5 msgs in a full segment + "touch": (True, 20., 1), + "can": (True, 100., 2053), # decimation gives ~3 msgs in a full segment "controlsState": (True, 100., 10), + "selfdriveState": (True, 100., 10), "pandaStates": (True, 10., 1), "peripheralState": (True, 2., 1), "radarState": (True, 20., 5), @@ -38,7 +40,8 @@ _services: dict[str, tuple] = { "carState": (True, 100., 10), "carControl": (True, 100., 10), "carOutput": (True, 100., 10), - "longitudinalPlan": (True, 20., 5), + "longitudinalPlan": (True, 20., 10), + "driverAssistance": (True, 20., 20), "procLog": (True, 0.5, 15), "gpsLocationExternal": (True, 10., 10), "gpsLocation": (True, 1., 1), @@ -47,9 +50,9 @@ _services: dict[str, tuple] = { "gnssMeasurements": (True, 10., 10), "clocks": (True, 0.1, 1), "ubloxRaw": (True, 20.), - "liveLocationKalman": (True, 20., 5), + "livePose": (True, 20., 4), "liveParameters": (True, 20., 5), - "cameraOdometry": (True, 20., 5), + "cameraOdometry": (True, 20., 10), "thumbnail": (True, 0.2, 1), "onroadEvents": (True, 1., 1), "carParams": (True, 0.02, 1), @@ -60,13 +63,13 @@ _services: dict[str, tuple] = { "driverMonitoringState": (True, 20., 10), "wideRoadEncodeIdx": (False, 20., 1), "wideRoadCameraState": (True, 20., 20), - "modelV2": (True, 20., 40), + "drivingModelData": (True, 20., 10), + "modelV2": (True, 20.), "managerState": (True, 2., 1), "uploaderState": (True, 0., 1), "navInstruction": (True, 1., 10), "navRoute": (True, 0.), "navThumbnail": (True, 0.), - "uiPlan": (True, 20., 40.), "qRoadEncodeIdx": (False, 20.), "userFlag": (True, 0., 1), "microphone": (True, 10., 10), @@ -74,6 +77,7 @@ _services: dict[str, tuple] = { # debug "uiDebug": (True, 0., 1), "testJoystick": (True, 0.), + "alertDebug": (True, 20., 5), "roadEncodeData": (False, 20.), "driverEncodeData": (False, 20.), "wideRoadEncodeData": (False, 20.), diff --git a/common/api/__init__.py b/common/api.py similarity index 93% rename from common/api/__init__.py rename to common/api.py index f3a7d83842..ac231400a4 100644 --- a/common/api/__init__.py +++ b/common/api.py @@ -1,7 +1,7 @@ import jwt import os import requests -from datetime import datetime, timedelta +from datetime import datetime, timedelta, UTC from openpilot.system.hardware.hw import Paths from openpilot.system.version import get_version @@ -23,7 +23,7 @@ class Api: return api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params) def get_token(self, expiry_hours=1): - now = datetime.utcnow() + now = datetime.now(UTC).replace(tzinfo=None) payload = { 'identity': self.dongle_id, 'nbf': now, diff --git a/common/clutil.cc b/common/clutil.cc index 4f2a783d3e..ee43ef36d5 100644 --- a/common/clutil.cc +++ b/common/clutil.cc @@ -79,6 +79,10 @@ cl_context cl_create_context(cl_device_id device_id) { return CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err)); } +void cl_release_context(cl_context context) { + clReleaseContext(context); +} + cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args) { return cl_program_from_source(ctx, device_id, util::read_file(path), args); } diff --git a/common/clutil.h b/common/clutil.h index af986d6434..c8bd2cd38f 100644 --- a/common/clutil.h +++ b/common/clutil.h @@ -23,6 +23,7 @@ cl_device_id cl_get_device_id(cl_device_type device_type); cl_context cl_create_context(cl_device_id device_id); +void cl_release_context(cl_context context); cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args = nullptr); cl_program cl_program_from_binary(cl_context ctx, cl_device_id device_id, const uint8_t* binary, size_t length, const char* args = nullptr); cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args); diff --git a/common/gps.py b/common/gps.py new file mode 100644 index 0000000000..6f96d72e99 --- /dev/null +++ b/common/gps.py @@ -0,0 +1,8 @@ +from openpilot.common.params import Params + + +def get_gps_location_service(params: Params) -> str: + if params.get_bool("UbloxAvailable"): + return "gpsLocationExternal" + else: + return "gpsLocation" diff --git a/common/markdown.py b/common/markdown.py new file mode 100644 index 0000000000..f0f056d963 --- /dev/null +++ b/common/markdown.py @@ -0,0 +1,45 @@ +HTML_REPLACEMENTS = [ + (r'&', r'&'), + (r'"', r'"'), +] + +def parse_markdown(text: str, tab_length: int = 2) -> str: + lines = text.split("\n") + output: list[str] = [] + list_level = 0 + + def end_outstanding_lists(level: int, end_level: int) -> int: + while level > end_level: + level -= 1 + output.append("") + if level > 0: + output.append("") + return end_level + + for i, line in enumerate(lines): + if i + 1 < len(lines) and lines[i + 1].startswith("==="): # heading + output.append(f"

{line}

") + elif line.startswith("==="): + pass + elif line.lstrip().startswith("* "): # list + line_level = 1 + line.count(" " * tab_length, 0, line.index("*")) + if list_level >= line_level: + list_level = end_outstanding_lists(list_level, line_level) + else: + list_level += 1 + if list_level > 1: + output[-1] = output[-1].replace("", "") + output.append("