Merge branch 'master' into ui_networking_fix_connecting_not_show

.devcontainer/.gitignore

@@ -0,0 +1 @@
+.Xauthority

.devcontainer/Dockerfile

@@ -0,0 +1,6 @@
+FROM ghcr.io/commaai/openpilot-base:latest
+
+# remove gitconfig if exists, since its gonna be replaced by host one
+RUN rm -f /root/.gitconfig
+RUN apt update && apt install -y vim net-tools usbutils htop ripgrep
+RUN pip install ipython jupyter jupyterlab

.devcontainer/devcontainer.json

@@ -0,0 +1,23 @@
+{
+  "name": "openpilot devcontainer",
+  "build": {
+    "dockerfile": "Dockerfile"
+  },
+  "postCreateCommand": "bash -c 'if [[ $DISPLAY == *xquartz* ]]; then echo \"export DISPLAY=host.docker.internal:0\" >> /root/.bashrc; fi'",
+  "postStartCommand": "git config --file .gitmodules --get-regexp path | awk '{ print $2 }' | xargs -I{} git config --global --add safe.directory \"$PWD/{}\"",
+  "initializeCommand": ".devcontainer/setup_host.sh",
+  "privileged": true,
+  "containerEnv": {
+    "DISPLAY": "${localEnv:DISPLAY}",
+    "PYTHONPATH": "${containerWorkspaceFolder}",
+    "force_color_prompt": "1"
+  },
+  "runArgs": [
+    "--volume=/tmp/.X11-unix:/tmp/.X11-unix",
+    "--volume=${localWorkspaceFolder}/.devcontainer/.Xauthority:/root/.Xauthority",
+    "--volume=${localEnv:HOME}/.comma:/root/.comma",
+    "--volume=/tmp/comma_download_cache:/tmp/comma_download_cache",
+    "--volume=/tmp/devcontainer_scons_cache:/tmp/scons_cache",
+    "--shm-size=1G"
+  ]
+}

.devcontainer/setup_host.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# setup links to Xauthority
+XAUTHORITY_LINK=".devcontainer/.Xauthority"
+rm -f $XAUTHORITY_LINK
+if [[ -z $XAUTHORITY ]]; then
+    echo "XAUTHORITY not set. Fallback to ~/.Xauthority ..."
+    if ! [[ -f $HOME/.Xauthority ]]; then
+        echo "~/.XAuthority file does not exist. GUI tools may not work properly."
+        touch $XAUTHORITY_LINK # dummy file to satisfy container volume mount
+    else
+        ln -sf $HOME/.Xauthority $XAUTHORITY_LINK
+    fi
+else
+    ln -sf $XAUTHORITY $XAUTHORITY_LINK
+fi

.dockerignore

@@ -16,6 +16,9 @@
 *.so
 *.a
 
+venv/
+.venv/
+
 notebooks
 phone
 massivemap

.github/workflows/badges.yaml

@@ -7,7 +7,7 @@ on:
 env:
   BASE_IMAGE: openpilot-base
   DOCKER_REGISTRY: ghcr.io/commaai
-  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v ~/scons_cache:/tmp/scons_cache -v ~/comma_download_cache:/tmp/comma_download_cache -v ~/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/sh -c
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/sh -c
 
 jobs:
   badges:

.github/workflows/prebuilt.yaml

@@ -5,12 +5,8 @@ on:
   workflow_dispatch:
 
 env:
-  BASE_IMAGE: openpilot-base
-  DOCKER_REGISTRY: ghcr.io/commaai
-
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: |
+  BUILD: selfdrive/test/docker_build.sh prebuilt
-      DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
 
 jobs:
   build_prebuilt:
@@ -18,7 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
     if: github.repository == 'commaai/openpilot'
     env:
-      IMAGE_NAME: openpilot-prebuilt
+      PUSH_IMAGE: true
     steps:
     - name: Wait for green check mark
       if: ${{ github.event_name != 'workflow_dispatch' }}
@@ -31,13 +27,7 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - name: Build Docker image
+    - name: Build and Push docker image
-      run: |
-        eval "$BUILD"
-        DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f Dockerfile.openpilot .
-    - name: Push to container registry
       run: |
         $DOCKER_LOGIN
-        docker push $DOCKER_REGISTRY/$IMAGE_NAME:latest
+        eval "$BUILD"
-        docker tag $DOCKER_REGISTRY/$IMAGE_NAME:latest $DOCKER_REGISTRY/$IMAGE_NAME:$GITHUB_SHA
-        docker push $DOCKER_REGISTRY/$IMAGE_NAME:$GITHUB_SHA

.github/workflows/selfdrive_tests.yaml

@@ -12,21 +12,18 @@ concurrency:
 
 env:
   PYTHONWARNINGS: error
-
   BASE_IMAGE: openpilot-base
   CL_BASE_IMAGE: openpilot-base-cl
-  DOCKER_REGISTRY: ghcr.io/commaai
   AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}
 
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: |
+  BUILD: selfdrive/test/docker_build.sh base
-      DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
+
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
 
-  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v ~/scons_cache:/tmp/scons_cache -v ~/comma_download_cache:/tmp/comma_download_cache -v ~/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
+  BUILD_CL: selfdrive/test/docker_build.sh cl
 
-  BUILD_CL: |
+  RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
-      DOCKER_BUILDKIT=1 docker build --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $CL_BASE_IMAGE:latest -f Dockerfile.openpilot_base_cl .
-  RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v ~/scons_cache:/tmp/scons_cache -v ~/comma_download_cache:/tmp/comma_download_cache -v ~/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
 
   UNIT_TEST: coverage run --append -m unittest discover
 
@@ -64,7 +61,6 @@ jobs:
       run: |
         cd $GITHUB_WORKSPACE
         cp .pre-commit-config.yaml $STRIPPED_DIR
-        cp mypy.ini $STRIPPED_DIR
         cp pyproject.toml $STRIPPED_DIR
         cp poetry.lock $STRIPPED_DIR
         cd $STRIPPED_DIR
@@ -80,7 +76,9 @@ jobs:
     - uses: ./.github/workflows/setup
     - name: Build openpilot with all flags
       timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 12 || 30) }} # allow more time when we missed the scons cache
-      run: ${{ env.RUN }} "scons -j$(nproc) --extras && release/check-dirty.sh"
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+        ${{ env.RUN }} "release/check-dirty.sh"
     - name: Cleanup scons cache and rebuild
       timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 2 || 30) }} # allow more time when we missed the scons cache
       run: |
@@ -90,7 +88,7 @@ jobs:
       uses: actions/cache/save@v3
       if: github.ref == 'refs/heads/master'
       with:
-        path: ~/scons_cache
+        path: .ci_cache/scons_cache
         key: scons-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
 
   build_mac:
@@ -118,6 +116,8 @@ jobs:
       uses: actions/cache@v3
       with:
         path: |
+          .env
+          .venv
           ~/github_brew_cache_entries.txt
           ~/.pyenv
           ~/Library/Caches/pypoetry
@@ -149,18 +149,17 @@ jobs:
         PYTHONWARNINGS: default
     - name: Build openpilot
       run: |
-        source tools/openpilot_env.sh
+        eval "$(pyenv init --path)"
         poetry run scons -j$(nproc)
     - name: Run tests
       run: |
-        source tools/openpilot_env.sh
+        eval "$(pyenv init --path)"
-        export PYTHONPATH=$PWD
         poetry run tools/plotjuggler/test_plotjuggler.py
     - name: Pre Cache - Cleanup scons cache
       if: github.ref == 'refs/heads/master'
       run: |
-        source tools/openpilot_env.sh
         rm -rf /tmp/scons_cache/*
+        eval "$(pyenv init --path)"
         poetry run scons -j$(nproc) --cache-populate
     - name: Save scons cache
       id: scons-save-cache
@@ -192,22 +191,16 @@ jobs:
     - uses: actions/checkout@v3
       with:
         submodules: true
-    - name: Build Docker image
+    - name: Setup to push to repo
-      run: eval "$BUILD"
-    - name: Push to container registry
       run: |
+        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
         $DOCKER_LOGIN
-        docker push $DOCKER_REGISTRY/$BASE_IMAGE:latest
+    - uses: ./.github/workflows/setup
-        docker tag $DOCKER_REGISTRY/$BASE_IMAGE:latest $DOCKER_REGISTRY/$BASE_IMAGE:$GITHUB_SHA
+      with:
-        docker push $DOCKER_REGISTRY/$BASE_IMAGE:$GITHUB_SHA
+        git-lfs: false
-    - name: Build CL Docker image
+    - name: Build and push CL Docker image
-      run: eval "$BUILD_CL"
-    - name: Push to container registry
       run: |
-        $DOCKER_LOGIN
+        eval "$BUILD_CL"
-        docker push $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest
-        docker tag $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest $DOCKER_REGISTRY/$CL_BASE_IMAGE:$GITHUB_SHA
-        docker push $DOCKER_REGISTRY/$CL_BASE_IMAGE:$GITHUB_SHA
 
   static_analysis:
     name: static analysis
@@ -276,6 +269,7 @@ jobs:
                         QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
                         ./selfdrive/ui/tests/test_translations.py && \
                         ./common/tests/test_util && \
+                        ./common/tests/test_ratekeeper && \
                         ./common/tests/test_swaglog && \
                         ./selfdrive/boardd/tests/test_boardd_usbprotocol && \
                         ./system/loggerd/tests/test_logger &&\
@@ -300,13 +294,13 @@ jobs:
       id: dependency-cache
       uses: actions/cache@v3
       with:
-        path: ~/comma_download_cache
+        path: .ci_cache/comma_download_cache
         key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit') }}
     - name: Build openpilot
       run: |
         ${{ env.RUN }} "scons -j$(nproc)"
     - name: Run replay
-      timeout-minutes: 20
+      timeout-minutes: 30
       run: |
         ${{ env.RUN }} "CI=1 coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
                         coverage xml"
@@ -323,7 +317,7 @@ jobs:
     - name: Upload reference logs
       if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
       run: |
-        ${{ env.RUN }} "CI=1 AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
+        ${{ env.RUN }} "unset PYTHONWARNINGS && CI=1 AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
     - name: "Upload coverage to Codecov"
       uses: codecov/codecov-action@v3
 
@@ -375,15 +369,14 @@ jobs:
       id: dependency-cache
       uses: actions/cache@v3
       with:
-        path: ~/comma_download_cache
+        path: .ci_cache/comma_download_cache
         key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }}
     - name: Build openpilot
       run: ${{ env.RUN }} "scons -j$(nproc)"
     - name: Test car models
       timeout-minutes: 25
       run: |
-        ${{ env.RUN }} "coverage run -m pytest selfdrive/car/tests/test_models.py && \
+        ${{ env.RUN }} "pytest --cov --cov-report=xml -n auto --dist=loadscope selfdrive/car/tests/test_models.py && \
-                        coverage xml && \
                         chmod -R 777 /tmp/comma_download_cache"
       env:
         NUM_JOBS: 5
@@ -408,9 +401,11 @@ jobs:
       - uses: actions/checkout@v3
         with:
           submodules: true
+          path: current
       - name: Save car docs diff
         id: save_diff
         run: |
+          cd current
           ${{ env.RUN }} "scons -j$(nproc)"
           output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_info")
           output="${output//$'\n'/'%0A'}"

.github/workflows/setup/action.yaml

@@ -1,6 +1,10 @@
 name: 'openpilot env setup'
 
 inputs:
+  setup_docker_scons_cache:
+    description: 'Whether or not to build the scons-cache docker image'
+    required: false
+    default: 'false'
   git_lfs:
     description: 'Whether or not to pull the git lfs'
     required: false
@@ -23,12 +27,31 @@ runs:
     - id: restore-scons-cache
       uses: actions/cache/restore@v3
       with:
-        path: ~/scons_cache
+        path: .ci_cache/scons_cache
         key: scons-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
         restore-keys: |
           scons-${{ env.CACHE_COMMIT_DATE }}-
           scons-
-
+    # if we didn't get a cache hit, make the directory manually so it doesn't fail on future steps
+    - id: scons-cache-setup
+      shell: bash
+      if: steps.restore-scons-cache.outputs.cache-hit != 'true'
+      run: mkdir -p $GITHUB_WORKSPACE/.ci_cache/scons_cache
+    # as suggested here: https://github.com/moby/moby/issues/32816#issuecomment-910030001
+    - id: normalize-file-permissions
+      shell: bash
+      name: Normalize file permissions to ensure a consistent docker build cache
+      run: |
+        find . -type f -executable -not -perm 755 -exec chmod 755 {} \;
+        find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \;
     # build our docker image
     - shell: bash
       run: eval ${{ env.BUILD }}
+    - id: setup-scons-cache-docker
+      name: Sets up a docker image with scons cache that can by mounted as a buildkit cache mount
+      shell: bash
+      if: ${{ inputs.setup_docker_scons_cache == 'true' }}
+      run: |
+        cp selfdrive/test/Dockerfile.scons_cache $GITHUB_WORKSPACE/.ci_cache
+        cd $GITHUB_WORKSPACE/.ci_cache
+        DOCKER_BUILDKIT=1 docker build -t scons-cache -f Dockerfile.scons_cache .

.github/workflows/tools_tests.yaml

@@ -13,17 +13,15 @@ concurrency:
 env:
   BASE_IMAGE: openpilot-base
   CL_BASE_IMAGE: openpilot-base-cl
-  DOCKER_REGISTRY: ghcr.io/commaai
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
 
-  BUILD: |
+  BUILD: selfdrive/test/docker_build.sh base
-      DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
 
-  RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v ~/scons_cache:/tmp/scons_cache -v ~/comma_download_cache:/tmp/comma_download_cache -v ~/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
+  RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
 
-  BUILD_CL: |
+  BUILD_CL: selfdrive/test/docker_build.sh cl
-      DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $CL_BASE_IMAGE:latest -f Dockerfile.openpilot_base_cl .
+
-  RUN_CL: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v ~/scons_cache:/tmp/scons_cache -v ~/comma_download_cache:/tmp/comma_download_cache -v ~/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
+  RUN_CL: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
 
 
 jobs:
@@ -37,20 +35,17 @@ jobs:
         submodules: true
     - name: Build Docker image
       run: eval "$BUILD"
-    - name: Unit test
+    - name: Build openpilot
+      timeout-minutes: 5
+      run: ${{ env.RUN }} "scons -j$(nproc) --directory=/tmp/openpilot/cereal --minimal"
+    - name: Test PlotJuggler
       timeout-minutes: 2
       run: |
-        ${{ env.RUN }} "scons -j$(nproc) --directory=/tmp/openpilot/cereal && \
+        ${{ env.RUN }} "pytest tools/plotjuggler/"
-                        apt-get update && \
-                        apt-get install -y libdw-dev libqt5svg5-dev libqt5x11extras5-dev && \
-                        cd /tmp/openpilot/tools/plotjuggler && \
-                        ./test_plotjuggler.py"
 
   simulator:
     name: simulator
     runs-on: ubuntu-20.04
-    env:
-      IMAGE_NAME: openpilot-sim
     if: github.repository == 'commaai/openpilot'
     timeout-minutes: 45
     steps:
@@ -58,34 +53,52 @@ jobs:
       with:
         submodules: true
     - uses: ./.github/workflows/setup
+      with:
+        setup_docker_scons_cache: true
     - name: Build base cl image
       run: eval "$BUILD_CL"
-    - name: Build simulator image
+    - name: Setup to push to repo
-      run: DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/sim/Dockerfile.sim .
-    - name: Push to container registry
       if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
       run: |
+        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
         $DOCKER_LOGIN
-        docker push $DOCKER_REGISTRY/$IMAGE_NAME:latest
+    - name: Build and push sim image
+      run: |
+        selfdrive/test/docker_build.sh sim
 
   docs:
     name: build docs
     runs-on: ubuntu-20.04
     timeout-minutes: 45
-    env:
-      BUILD: |
-        DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/openpilot-docs-base:latest -t $DOCKER_REGISTRY/openpilot-docs-base:latest -f docs/docker/Dockerfile --target openpilot-docs-base .
-        DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/openpilot-docs-base:latest --cache-from $DOCKER_REGISTRY/openpilot-docs:latest -t $DOCKER_REGISTRY/openpilot-docs:latest -f docs/docker/Dockerfile .
     steps:
     - uses: actions/checkout@v3
       with:
         submodules: true
     - uses: ./.github/workflows/setup
       with:
+        setup_docker_scons_cache: true
         git_lfs: false
-    - name: Push docker container
+    - name: Setup to push to repo
       if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
       run: |
+        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
         $DOCKER_LOGIN
-        docker push $DOCKER_REGISTRY/openpilot-docs-base:latest
+    - name: Build and push docs image
-        docker push $DOCKER_REGISTRY/openpilot-docs:latest
+      run: |
+        selfdrive/test/docker_build.sh docs
+
+  devcontainer:
+    name: devcontainer
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+    - name: Setup Dev Container CLI
+      run: npm install -g @devcontainers/cli
+    - name: Build dev container image
+      run: devcontainer build --workspace-folder .
+    - name: Run dev container
+      run: devcontainer up --workspace-folder .
+    - name: Test environment
+      run: devcontainer exec --workspace-folder . scons --dry-run

.gitignore

@@ -1,4 +1,6 @@
 venv/
+.venv/
+.ci_cache
 .env
 .clang-format
 .DS_Store

.pre-commit-config.yaml

@@ -35,7 +35,7 @@ repos:
         args: ['--explicit-package-bases']
         exclude: '^(third_party/)|(cereal/)|(opendbc/)|(panda/)|(laika/)|(laika_repo/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(xx/)'
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.0.284
+    rev: v0.0.285
     hooks:
     -   id: ruff
         exclude: '^(third_party/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(laika_repo/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)'
@@ -53,6 +53,18 @@ repos:
         - --quiet
         - --force
         - -j8
+-   repo: https://github.com/cpplint/cpplint
+    rev: 1.6.1
+    hooks:
+    -   id: cpplint
+        exclude: '^(third_party/)|(cereal/)|(body/)|(rednose/)|(rednose_repo/)|(opendbc/)|(panda/)|(generated/)'
+        args:
+        - --quiet
+        - --counting=total
+        - --linelength=240
+        # https://google.github.io/styleguide/cppguide.html
+        # relevant rules are whitelisted, see all options with: cpplint --filter=
+        - --filter=-build,-legal,-readability,-runtime,-whitespace,+build/include_subdir,+build/forward_decl,+build/include_what_you_use,+build/deprecated,+whitespace/comma,+whitespace/line_length,+whitespace/empty_if_body,+whitespace/empty_loop_body,+whitespace/empty_conditional_body,+whitespace/forcolon,+whitespace/parens,+whitespace/semicolon,+whitespace/tab,+readability/braces
 -   repo: local
     hooks:
     -   id: test_translations
@@ -61,10 +73,10 @@ repos:
         language: script
         pass_filenames: false
 -   repo: https://github.com/python-poetry/poetry
-    rev: '1.5.0'
+    rev: '1.6.0'
     hooks:
     -   id: poetry-check
     -   id: poetry-lock
         name: validate poetry lock
-        args: 
+        args:
         - --check

Dockerfile.openpilot

@@ -10,6 +10,7 @@ WORKDIR ${OPENPILOT_PATH}
 
 COPY SConstruct ${OPENPILOT_PATH}
 
+COPY ./openpilot ${OPENPILOT_PATH}/openpilot
 COPY ./third_party ${OPENPILOT_PATH}/third_party
 COPY ./site_scons ${OPENPILOT_PATH}/site_scons
 COPY ./laika ${OPENPILOT_PATH}/laika

Dockerfile.openpilot_base

@@ -18,7 +18,7 @@ ENV PYENV_ROOT="/root/.pyenv"
 ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH"
 
 COPY pyproject.toml poetry.lock .python-version /tmp/
-COPY tools/ubuntu_setup.sh tools/install_python_dependencies.sh /tmp/tools/
+COPY tools/ubuntu_setup.sh tools/install_ubuntu_dependencies.sh tools/install_python_dependencies.sh /tmp/tools/
 RUN cd /tmp && \
     tools/ubuntu_setup.sh && \
     rm -rf /var/lib/apt/lists/* && \

Jenkinsfile

@@ -160,6 +160,7 @@ pipeline {
             sh "scons --clean && scons --no-cache --random -j42"
             sh "INTERNAL_SEG_CNT=500 INTERNAL_SEG_LIST=selfdrive/car/tests/test_models_segs.txt FILEREADER_CACHE=1 \
                 pytest -n42 --dist=loadscope selfdrive/car/tests/test_models.py"
+            sh "MAX_EXAMPLES=100 pytest -n42 selfdrive/car/tests/test_car_interfaces.py"
           }
 
           post {

RELEASES.md

@@ -1,5 +1,6 @@
 Version 0.9.5 (202X-XX-XX)
 ========================
+* Kia Sorento Hybrid 2023 support thanks to sunnyhaibin!
 * Lexus IS 2023 support thanks to L3R5!
 
 Version 0.9.4 (2023-07-27)

SConstruct

@@ -14,10 +14,6 @@ AGNOS = TICI
 
 Decider('MD5-timestamp')
 
-AddOption('--extras',
-          action='store_true',
-          help='build misc extras, like setup and installer files')
-
 AddOption('--kaitai',
           action='store_true',
           help='Regenerate kaitai struct parsers')
@@ -58,11 +54,11 @@ AddOption('--pc-thneed',
           dest='pc_thneed',
           help='use thneed on pc')
 
-AddOption('--no-test',
+AddOption('--minimal',
           action='store_false',
-          dest='test',
+          dest='extras',
           default=os.path.islink(Dir('#laika/').abspath),
-          help='skip building test files')
+          help='the minimum build to run openpilot. no tests, tools, etc.')
 
 ## Architecture name breakdown (arch)
 ## - larch64: linux tici aarch64
@@ -334,7 +330,6 @@ qt_flags = [
 qt_env['CXXFLAGS'] += qt_flags
 qt_env['LIBPATH'] += ['#selfdrive/ui']
 qt_env['LIBS'] = qt_libs
-qt_env['QT3_MOCHPREFIX'] = cache_dir + '/moc_files/moc_'
 
 if GetOption("clazy"):
   checks = [
@@ -437,7 +432,7 @@ SConscript(['selfdrive/navd/SConscript'])
 SConscript(['selfdrive/modeld/SConscript'])
 SConscript(['selfdrive/ui/SConscript'])
 
-if (arch in ['x86_64', 'aarch64', 'Darwin'] and Dir('#tools/cabana/').exists()) or GetOption('extras'):
+if arch in ['x86_64', 'aarch64', 'Darwin'] and Dir('#tools/cabana/').exists() and GetOption('extras'):
   SConscript(['tools/replay/SConscript'])
   SConscript(['tools/cabana/SConscript'])
 

cereal

@@ -1 +1 @@
-Subproject commit 2077dd1a1b0054713102aee8197434f8cfa07de7
+Subproject commit 82bca3a9714b73c05414fdf848b6016a0ffac17d

common/SConscript

@@ -12,6 +12,7 @@ common_libs = [
   'util.cc',
   'i2c.cc',
   'watchdog.cc',
+  'ratekeeper.cc'
 ]
 
 if arch != "Darwin":
@@ -26,10 +27,10 @@ files = [
 _gpucommon = fxn('gpucommon', files)
 Export('_common', '_gpucommon')
 
-if GetOption('test'):
+if GetOption('extras'):
   env.Program('tests/test_util', ['tests/test_util.cc'], LIBS=[_common])
   env.Program('tests/test_swaglog', ['tests/test_swaglog.cc'], LIBS=[_common, 'json11', 'zmq', 'pthread'])
+  env.Program('tests/test_ratekeeper', ['tests/test_ratekeeper.cc'], LIBS=[_common, 'json11', 'zmq', 'pthread'])
 
 # Cython
-envCython.Program('clock.so', 'clock.pyx')
 envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])

common/api/__init__.py

@@ -2,8 +2,8 @@ import jwt
 import os
 import requests
 from datetime import datetime, timedelta
-from common.basedir import PERSIST
+from openpilot.common.basedir import PERSIST
-from system.version import get_version
+from openpilot.system.version import get_version
 
 API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com')
 

common/basedir.py

@@ -1,7 +1,7 @@
 import os
 from pathlib import Path
 
-from system.hardware import PC
+from openpilot.system.hardware import PC
 
 BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../"))
 

common/clock.pyx

@@ -1,24 +0,0 @@
-# distutils: language = c++
-# cython: language_level = 3
-from posix.time cimport clock_gettime, timespec, CLOCK_MONOTONIC_RAW, clockid_t
-
-IF UNAME_SYSNAME == "Darwin":
-  # Darwin doesn't have a CLOCK_BOOTTIME
-  CLOCK_BOOTTIME = CLOCK_MONOTONIC_RAW
-ELSE:
-  from posix.time cimport CLOCK_BOOTTIME
-
-cdef double readclock(clockid_t clock_id):
-  cdef timespec ts
-  cdef double current
-
-  clock_gettime(clock_id, &ts)
-  current = ts.tv_sec + (ts.tv_nsec / 1000000000.)
-  return current
-
-def monotonic_time():
-  return readclock(CLOCK_MONOTONIC_RAW)
-
-def sec_since_boot():
-  return readclock(CLOCK_BOOTTIME)
-

common/clutil.cc

@@ -75,6 +75,10 @@ cl_device_id cl_get_device_id(cl_device_type device_type) {
   return nullptr;
 }
 
+cl_context cl_create_context(cl_device_id device_id) {
+  return CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err));
+}
+
 cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args) {
   return cl_program_from_source(ctx, device_id, util::read_file(path), args);
 }

common/clutil.h

@@ -22,6 +22,7 @@
   })
 
 cl_device_id cl_get_device_id(cl_device_type device_type);
+cl_context cl_create_context(cl_device_id device_id);
 cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args = nullptr);
 cl_program cl_program_from_binary(cl_context ctx, cl_device_id device_id, const uint8_t* binary, size_t length, const char* args = nullptr);
 cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args);

common/gpio.cc

@@ -1,5 +1,7 @@
 #include "common/gpio.h"
 
+#include <string>
+
 #ifdef __APPLE__
 int gpio_init(int pin_nr, bool output) {
   return 0;
@@ -29,7 +31,7 @@ int gpio_init(int pin_nr, bool output) {
   char pin_dir_path[50];
   int pin_dir_path_len = snprintf(pin_dir_path, sizeof(pin_dir_path),
                            "/sys/class/gpio/gpio%d/direction", pin_nr);
-  if(pin_dir_path_len <= 0) {
+  if (pin_dir_path_len <= 0) {
     return -1;
   }
   const char *value = output ? "out" : "in";
@@ -40,7 +42,7 @@ int gpio_set(int pin_nr, bool high) {
   char pin_val_path[50];
   int pin_val_path_len = snprintf(pin_val_path, sizeof(pin_val_path),
                            "/sys/class/gpio/gpio%d/value", pin_nr);
-  if(pin_val_path_len <= 0) {
+  if (pin_val_path_len <= 0) {
     return -1;
   }
   return util::write_file(pin_val_path, (void*)(high ? "1" : "0"), 1);

common/gpio.py

@@ -1,3 +1,4 @@
+import os
 from functools import lru_cache
 from typing import Optional, List
 
@@ -26,6 +27,9 @@ def gpio_read(pin: int) -> Optional[bool]:
   return val
 
 def gpio_export(pin: int) -> None:
+  if os.path.isdir(f"/sys/class/gpio/gpio{pin}"):
+    return
+
   try:
     with open("/sys/class/gpio/export", 'w') as f:
       f.write(str(pin))

common/i2c.cc

@@ -8,7 +8,6 @@
 #include <cstdio>
 #include <stdexcept>
 
-#include "common/util.h"
 #include "common/swaglog.h"
 #include "common/util.h"
 
@@ -26,23 +25,23 @@ I2CBus::I2CBus(uint8_t bus_id) {
   snprintf(bus_name, 20, "/dev/i2c-%d", bus_id);
 
   i2c_fd = HANDLE_EINTR(open(bus_name, O_RDWR));
-  if(i2c_fd < 0) {
+  if (i2c_fd < 0) {
     throw std::runtime_error("Failed to open I2C bus");
   }
 }
 
 I2CBus::~I2CBus() {
-  if(i2c_fd >= 0) { close(i2c_fd); }
+  if (i2c_fd >= 0) { close(i2c_fd); }
 }
 
 int I2CBus::read_register(uint8_t device_address, uint register_address, uint8_t *buffer, uint8_t len) {
   int ret = 0;
 
   ret = HANDLE_EINTR(ioctl(i2c_fd, I2C_SLAVE, device_address));
-  if(ret < 0) { goto fail; }
+  if (ret < 0) { goto fail; }
 
   ret = i2c_smbus_read_i2c_block_data(i2c_fd, register_address, len, buffer);
-  if((ret < 0) || (ret != len)) { goto fail; }
+  if ((ret < 0) || (ret != len)) { goto fail; }
 
 fail:
   return ret;
@@ -52,10 +51,10 @@ int I2CBus::set_register(uint8_t device_address, uint register_address, uint8_t
   int ret = 0;
 
   ret = HANDLE_EINTR(ioctl(i2c_fd, I2C_SLAVE, device_address));
-  if(ret < 0) { goto fail; }
+  if (ret < 0) { goto fail; }
 
   ret = i2c_smbus_write_byte_data(i2c_fd, register_address, data);
-  if(ret < 0) { goto fail; }
+  if (ret < 0) { goto fail; }
 
 fail:
   return ret;

common/kalman/simple_kalman.py

@@ -1,3 +1,12 @@
-# pylint: skip-file
+from openpilot.common.kalman.simple_kalman_impl import KF1D as KF1D
-from common.kalman.simple_kalman_impl import KF1D as KF1D
 assert KF1D
+import numpy as np
+
+def get_kalman_gain(dt, A, C, Q, R, iterations=100):
+  P = np.zeros_like(Q)
+  for _ in range(iterations):
+    P = A.dot(P).dot(A.T) + dt * Q
+    S = C.dot(P).dot(C.T) + R
+    K = P.dot(C.T).dot(np.linalg.inv(S))
+    P = (np.eye(len(P)) - K.dot(C)).dot(P)
+  return K

common/kalman/tests/test_simple_kalman.py

@@ -3,8 +3,8 @@ import random
 import timeit
 import numpy as np
 
-from common.kalman.simple_kalman import KF1D
+from openpilot.common.kalman.simple_kalman import KF1D
-from common.kalman.simple_kalman_old import KF1D as KF1D_old
+from openpilot.common.kalman.simple_kalman_old import KF1D as KF1D_old
 
 
 class TestSimpleKalman(unittest.TestCase):
@@ -54,8 +54,8 @@ class TestSimpleKalman(unittest.TestCase):
     setup = """
 import numpy as np
 
-from common.kalman.simple_kalman import KF1D
+from openpilot.common.kalman.simple_kalman import KF1D
-from common.kalman.simple_kalman_old import KF1D as KF1D_old
+from openpilot.common.kalman.simple_kalman_old import KF1D as KF1D_old
 
 dt = 0.01
 x0_0 = 0.0

common/logging_extra.py

@@ -197,7 +197,7 @@ class SwagLogger(logging.Logger):
       filename = os.path.normcase(co.co_filename)
 
       # TODO: is this pylint exception correct?
-      if filename == _srcfile:  # pylint: disable=comparison-with-callable
+      if filename == _srcfile:
         f = f.f_back
         continue
       sinfo = None

common/mat.h

@@ -1,7 +1,7 @@
 #pragma once
 
 typedef struct vec3 {
-	float v[3];
+  float v[3];
 } vec3;
 
 typedef struct vec4 {
@@ -9,7 +9,7 @@ typedef struct vec4 {
 } vec4;
 
 typedef struct mat3 {
-	float v[3*3];
+  float v[3*3];
 } mat3;
 
 typedef struct mat4 {

common/params.py

@@ -1,4 +1,5 @@
-from common.params_pyx import Params, ParamKeyType, UnknownKeyName, put_nonblocking, put_bool_nonblocking # pylint: disable=no-name-in-module, import-error
+from openpilot.common.params_pyx import Params, ParamKeyType, UnknownKeyName, put_nonblocking, \
+                                        put_bool_nonblocking
 assert Params
 assert ParamKeyType
 assert UnknownKeyName

common/ratekeeper.cc

@@ -0,0 +1,40 @@
+#include "common/ratekeeper.h"
+
+#include <algorithm>
+
+#include "common/swaglog.h"
+#include "common/timing.h"
+#include "common/util.h"
+
+RateKeeper::RateKeeper(const std::string &name, float rate, float print_delay_threshold)
+    : name(name),
+      print_delay_threshold(std::max(0.f, print_delay_threshold)) {
+  interval = 1 / rate;
+  last_monitor_time = seconds_since_boot();
+  next_frame_time = last_monitor_time + interval;
+}
+
+bool RateKeeper::keepTime() {
+  bool lagged = monitorTime();
+  if (remaining_ > 0) {
+    util::sleep_for(remaining_ * 1000);
+  }
+  return lagged;
+}
+
+bool RateKeeper::monitorTime() {
+  ++frame_;
+  last_monitor_time = seconds_since_boot();
+  remaining_ = next_frame_time - last_monitor_time;
+
+  bool lagged = remaining_ < 0;
+  if (lagged) {
+    if (print_delay_threshold > 0 && remaining_ < -print_delay_threshold) {
+      LOGW("%s lagging by %.2f ms", name.c_str(), -remaining_ * 1000);
+    }
+    next_frame_time = last_monitor_time + interval;
+  } else {
+    next_frame_time += interval;
+  }
+  return lagged;
+}

common/ratekeeper.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <cstdint>
+#include <string>
+
+class RateKeeper {
+public:
+  RateKeeper(const std::string &name, float rate, float print_delay_threshold = 0);
+  ~RateKeeper() {}
+  bool keepTime();
+  bool monitorTime();
+  inline double frame() const { return frame_; }
+  inline double remaining() const { return remaining_; }
+
+private:
+  double interval;
+  double next_frame_time;
+  double last_monitor_time;
+  double remaining_ = 0;
+  float print_delay_threshold = 0;
+  uint64_t frame_ = 0;
+  std::string name;
+};

common/realtime.py

@@ -5,10 +5,9 @@ import time
 from collections import deque
 from typing import Optional, List, Union
 
-from setproctitle import getproctitle  # pylint: disable=no-name-in-module
+from setproctitle import getproctitle
 
-from common.clock import sec_since_boot  # pylint: disable=no-name-in-module, import-error
+from openpilot.system.hardware import PC
-from system.hardware import PC
 
 
 # time step for each process
@@ -31,12 +30,12 @@ class Priority:
 
 def set_realtime_priority(level: int) -> None:
   if not PC:
-    os.sched_setscheduler(0, os.SCHED_FIFO, os.sched_param(level))  # pylint: disable=no-member
+    os.sched_setscheduler(0, os.SCHED_FIFO, os.sched_param(level))
 
 
 def set_core_affinity(cores: List[int]) -> None:
   if not PC:
-    os.sched_setaffinity(0, cores)  # pylint: disable=no-member
+    os.sched_setaffinity(0, cores)
 
 
 def config_realtime_process(cores: Union[int, List[int]], priority: int) -> None:
@@ -50,13 +49,13 @@ class Ratekeeper:
   def __init__(self, rate: float, print_delay_threshold: Optional[float] = 0.0) -> None:
     """Rate in Hz for ratekeeping. print_delay_threshold must be nonnegative."""
     self._interval = 1. / rate
-    self._next_frame_time = sec_since_boot() + self._interval
+    self._next_frame_time = time.monotonic() + self._interval
     self._print_delay_threshold = print_delay_threshold
     self._frame = 0
     self._remaining = 0.0
     self._process_name = getproctitle()
     self._dts = deque([self._interval], maxlen=100)
-    self._last_monitor_time = sec_since_boot()
+    self._last_monitor_time = time.monotonic()
 
   @property
   def frame(self) -> int:
@@ -82,11 +81,11 @@ class Ratekeeper:
   # this only monitor the cumulative lag, but does not enforce a rate
   def monitor_time(self) -> bool:
     prev = self._last_monitor_time
-    self._last_monitor_time = sec_since_boot()
+    self._last_monitor_time = time.monotonic()
     self._dts.append(self._last_monitor_time - prev)
 
     lagged = False
-    remaining = self._next_frame_time - sec_since_boot()
+    remaining = self._next_frame_time - time.monotonic()
     self._next_frame_time += self._interval
     if self._print_delay_threshold is not None and remaining < -self._print_delay_threshold:
       print(f"{self._process_name} lagging by {-remaining * 1000:.2f} ms")

common/spinner.py

@@ -1,6 +1,6 @@
 import os
 import subprocess
-from common.basedir import BASEDIR
+from openpilot.common.basedir import BASEDIR
 
 
 class Spinner():

common/swaglog.cc

@@ -12,7 +12,7 @@
 #include <string>
 
 #include <zmq.h>
-#include "json11.hpp"
+#include "third_party/json11/json11.hpp"
 
 #include "common/util.h"
 #include "common/version.h"
@@ -64,8 +64,7 @@ static void log(int levelnum, const char* filename, int lineno, const char* func
   if (levelnum >= s.print_level) {
     printf("%s: %s\n", filename, msg);
   }
-  char levelnum_c = levelnum;
+  zmq_send(s.sock, log_s.data(), log_s.length(), ZMQ_NOBLOCK);
-  zmq_send(s.sock, (levelnum_c + log_s).c_str(), log_s.length() + 1, ZMQ_NOBLOCK);
 }
 
 static void cloudlog_common(int levelnum, const char* filename, int lineno, const char* func,
@@ -87,8 +86,11 @@ static void cloudlog_common(int levelnum, const char* filename, int lineno, cons
     log_j["msg"] = msg_j;
   }
 
-  std::string log_s = ((json11::Json)log_j).dump();
+  std::string log_s;
+  log_s += (char)levelnum;
+  ((json11::Json)log_j).dump(log_s);
   log(levelnum, filename, lineno, func, msg_buf, log_s);
+
   free(msg_buf);
 }
 

common/swaglog.h

@@ -44,7 +44,7 @@ void cloudlog_te(int levelnum, const char* filename, int lineno, const char* fun
   int __millis = (millis);                          \
   uint64_t __ts = nanos_since_boot();               \
                                                     \
-  if (!__begin) __begin = __ts;                     \
+  if (!__begin) { __begin = __ts; }                 \
                                                     \
   if (__begin + __millis*1000000ULL < __ts) {       \
     if (__missed) {                                 \

common/tests/.gitignore

@@ -1,2 +1,3 @@
+test_ratekeeper
 test_util
 test_swaglog

common/tests/test_file_helpers.py

@@ -2,8 +2,8 @@ import os
 import unittest
 from uuid import uuid4
 
-from common.file_helpers import atomic_write_on_fs_tmp
+from openpilot.common.file_helpers import atomic_write_on_fs_tmp
-from common.file_helpers import atomic_write_in_dir
+from openpilot.common.file_helpers import atomic_write_in_dir
 
 
 class TestFileHelpers(unittest.TestCase):

common/tests/test_numpy_fast.py

@@ -1,7 +1,7 @@
 import numpy as np
 import unittest
 
-from common.numpy_fast import interp
+from openpilot.common.numpy_fast import interp
 
 
 class InterpTest(unittest.TestCase):

common/tests/test_params.py

@@ -6,7 +6,7 @@ import shutil
 import uuid
 import unittest
 
-from common.params import Params, ParamKeyType, UnknownKeyName, put_nonblocking, put_bool_nonblocking
+from openpilot.common.params import Params, ParamKeyType, UnknownKeyName, put_nonblocking, put_bool_nonblocking
 
 class TestParams(unittest.TestCase):
   def setUp(self):

common/tests/test_ratekeeper.cc

@@ -0,0 +1,17 @@
+#define CATCH_CONFIG_MAIN
+#include "catch2/catch.hpp"
+#include "common/ratekeeper.h"
+#include "common/timing.h"
+#include "common/util.h"
+
+TEST_CASE("RateKeeper") {
+  float freq = GENERATE(10, 50, 100);
+  RateKeeper rk("Test RateKeeper", freq);
+  for (int i = 0; i < freq; ++i) {
+    double begin = seconds_since_boot();
+    util::sleep_for(util::random_int(0, 1000.0 / freq - 1));
+    bool lagged = rk.keepTime();
+    REQUIRE(std::abs(seconds_since_boot() - begin - (1 / freq)) < 1e-3);
+    REQUIRE(lagged == false);
+  }
+}

common/tests/test_swaglog.cc

@@ -3,7 +3,7 @@
 #define CATCH_CONFIG_MAIN
 #include "catch2/catch.hpp"
 
-#include "json11.hpp"
+#include "third_party/json11/json11.hpp"
 #include "common/swaglog.h"
 #include "common/util.h"
 #include "common/version.h"

common/text_window.py

@@ -2,7 +2,7 @@
 import os
 import time
 import subprocess
-from common.basedir import BASEDIR
+from openpilot.common.basedir import BASEDIR
 
 
 class TextWindow:

common/transformations/camera.py

@@ -1,6 +1,6 @@
 import numpy as np
 
-import common.transformations.orientation as orient
+import openpilot.common.transformations.orientation as orient
 
 ## -- hardcoded hardware params --
 eon_f_focal_length = 910.0

common/transformations/coordinates.cc

@@ -1,13 +1,11 @@
 #define _USE_MATH_DEFINES
 
+#include "common/transformations/coordinates.hpp"
+
 #include <iostream>
 #include <cmath>
 #include <eigen3/Eigen/Dense>
 
-#include "coordinates.hpp"
-
-
-
 double a = 6378137; // lgtm [cpp/short-global-name]
 double b = 6356752.3142; // lgtm [cpp/short-global-name]
 double esq = 6.69437999014 * 0.001; // lgtm [cpp/short-global-name]

common/transformations/coordinates.hpp

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <eigen3/Eigen/Dense>
+
 #define DEG2RAD(x) ((x) * M_PI / 180.0)
 #define RAD2DEG(x) ((x) * 180.0 / M_PI)
 

common/transformations/coordinates.py

@@ -1,8 +1,7 @@
-# pylint: skip-file
+from openpilot.common.transformations.orientation import numpy_wrap
-from common.transformations.orientation import numpy_wrap
+from openpilot.common.transformations.transformations import (ecef2geodetic_single,
-from common.transformations.transformations import (ecef2geodetic_single,
                                                     geodetic2ecef_single)
-from common.transformations.transformations import LocalCoord as LocalCoord_single
+from openpilot.common.transformations.transformations import LocalCoord as LocalCoord_single
 
 
 class LocalCoord(LocalCoord_single):

common/transformations/model.py

@@ -1,6 +1,6 @@
 import numpy as np
 
-from common.transformations.camera import (FULL_FRAME_SIZE,
+from openpilot.common.transformations.camera import (FULL_FRAME_SIZE,
                                            get_view_frame_from_calib_frame)
 
 # segnet
@@ -61,8 +61,8 @@ medmodel_frame_from_bigmodel_frame = np.dot(medmodel_intrinsics, np.linalg.inv(b
 ### This function mimics the update_calibration logic in modeld.cc
 ### Manually verified to give similar results to xx.uncommon.utils.transform_img
 def get_warp_matrix(rpy_calib, wide_cam=False, big_model=False, tici=True):
-  from common.transformations.orientation import rot_from_euler
+  from openpilot.common.transformations.orientation import rot_from_euler
-  from common.transformations.camera import view_frame_from_device_frame, eon_fcam_intrinsics, tici_ecam_intrinsics, tici_fcam_intrinsics
+  from openpilot.common.transformations.camera import view_frame_from_device_frame, eon_fcam_intrinsics, tici_ecam_intrinsics, tici_fcam_intrinsics
 
   if tici and wide_cam:
     intrinsics = tici_ecam_intrinsics
@@ -85,8 +85,8 @@ def get_warp_matrix(rpy_calib, wide_cam=False, big_model=False, tici=True):
 
 ### This is old, just for debugging
 def get_warp_matrix_old(rpy_calib, wide_cam=False, big_model=False, tici=True):
-  from common.transformations.orientation import rot_from_euler
+  from openpilot.common.transformations.orientation import rot_from_euler
-  from common.transformations.camera import view_frame_from_device_frame, eon_fcam_intrinsics, tici_ecam_intrinsics, tici_fcam_intrinsics
+  from openpilot.common.transformations.camera import view_frame_from_device_frame, eon_fcam_intrinsics, tici_ecam_intrinsics, tici_fcam_intrinsics
 
 
   def get_view_frame_from_road_frame(roll, pitch, yaw, height):

common/transformations/orientation.cc

@@ -4,8 +4,8 @@
 #include <cmath>
 #include <eigen3/Eigen/Dense>
 
-#include "orientation.hpp"
+#include "common/transformations/orientation.hpp"
-#include "coordinates.hpp"
+#include "common/transformations/coordinates.hpp"
 
 Eigen::Quaterniond ensure_unique(Eigen::Quaterniond quat){
   if (quat.w() > 0){

common/transformations/orientation.hpp

@@ -1,6 +1,6 @@
 #pragma once
 #include <eigen3/Eigen/Dense>
-#include "coordinates.hpp"
+#include "common/transformations/coordinates.hpp"
 
 
 Eigen::Quaterniond ensure_unique(Eigen::Quaterniond quat);

common/transformations/orientation.py

@@ -1,8 +1,7 @@
-# pylint: skip-file
 import numpy as np
 from typing import Callable
 
-from common.transformations.transformations import (ecef_euler_from_ned_single,
+from openpilot.common.transformations.transformations import (ecef_euler_from_ned_single,
                                                     euler2quat_single,
                                                     euler2rot_single,
                                                     ned_euler_from_ecef_single,

common/transformations/tests/test_coordinates.py

@@ -3,7 +3,7 @@
 import numpy as np
 import unittest
 
-import common.transformations.coordinates as coord
+import openpilot.common.transformations.coordinates as coord
 
 geodetic_positions = np.array([[37.7610403, -122.4778699, 115],
                                  [27.4840915, -68.5867592, 2380],

common/transformations/tests/test_orientation.py

@@ -3,7 +3,7 @@
 import numpy as np
 import unittest
 
-from common.transformations.orientation import euler2quat, quat2euler, euler2rot, rot2euler, \
+from openpilot.common.transformations.orientation import euler2quat, quat2euler, euler2rot, rot2euler, \
                                                rot2quat, quat2rot, \
                                                ned_euler_from_ecef
 

common/transformations/transformations.pyx

@@ -1,20 +1,20 @@
 # distutils: language = c++
 # cython: language_level = 3
-from common.transformations.transformations cimport Matrix3, Vector3, Quaternion
+from openpilot.common.transformations.transformations cimport Matrix3, Vector3, Quaternion
-from common.transformations.transformations cimport ECEF, NED, Geodetic
+from openpilot.common.transformations.transformations cimport ECEF, NED, Geodetic
-
+
-from common.transformations.transformations cimport euler2quat as euler2quat_c
+from openpilot.common.transformations.transformations cimport euler2quat as euler2quat_c
-from common.transformations.transformations cimport quat2euler as quat2euler_c
+from openpilot.common.transformations.transformations cimport quat2euler as quat2euler_c
-from common.transformations.transformations cimport quat2rot as quat2rot_c
+from openpilot.common.transformations.transformations cimport quat2rot as quat2rot_c
-from common.transformations.transformations cimport rot2quat as rot2quat_c
+from openpilot.common.transformations.transformations cimport rot2quat as rot2quat_c
-from common.transformations.transformations cimport euler2rot as euler2rot_c
+from openpilot.common.transformations.transformations cimport euler2rot as euler2rot_c
-from common.transformations.transformations cimport rot2euler as rot2euler_c
+from openpilot.common.transformations.transformations cimport rot2euler as rot2euler_c
-from common.transformations.transformations cimport rot_matrix as rot_matrix_c
+from openpilot.common.transformations.transformations cimport rot_matrix as rot_matrix_c
-from common.transformations.transformations cimport ecef_euler_from_ned as ecef_euler_from_ned_c
+from openpilot.common.transformations.transformations cimport ecef_euler_from_ned as ecef_euler_from_ned_c
-from common.transformations.transformations cimport ned_euler_from_ecef as ned_euler_from_ecef_c
+from openpilot.common.transformations.transformations cimport ned_euler_from_ecef as ned_euler_from_ecef_c
-from common.transformations.transformations cimport geodetic2ecef as geodetic2ecef_c
+from openpilot.common.transformations.transformations cimport geodetic2ecef as geodetic2ecef_c
-from common.transformations.transformations cimport ecef2geodetic as ecef2geodetic_c
+from openpilot.common.transformations.transformations cimport ecef2geodetic as ecef2geodetic_c
-from common.transformations.transformations cimport LocalCoord_c
+from openpilot.common.transformations.transformations cimport LocalCoord_c
 
 
 import cython

common/util.cc

@@ -253,6 +253,14 @@ std::string dir_name(std::string const &path) {
   return path.substr(0, pos);
 }
 
+bool starts_with(const std::string &s1, const std::string &s2) {
+  return strncmp(s1.c_str(), s2.c_str(), s2.size()) == 0;
+}
+
+bool ends_with(const std::string &s1, const std::string &s2) {
+  return strcmp(s1.c_str() + (s1.size() - s2.size()), s2.c_str()) == 0;
+}
+
 std::string check_output(const std::string& command) {
   char buffer[128];
   std::string result;

common/util.h

@@ -77,6 +77,8 @@ float getenv(const char* key, float default_val);
 
 std::string hexdump(const uint8_t* in, const size_t size);
 std::string dir_name(std::string const& path);
+bool starts_with(const std::string &s1, const std::string &s2);
+bool ends_with(const std::string &s1, const std::string &s2);
 
 // ***** random helpers *****
 int random_int(int min, int max);
@@ -115,7 +117,7 @@ public:
 #ifndef __APPLE__
     std::signal(SIGPWR, (sighandler_t)set_do_exit);
 #endif
-  };
+  }
   inline static std::atomic<bool> power_failure = false;
   inline static std::atomic<int> signal = 0;
   inline operator bool() { return do_exit; }
@@ -151,12 +153,18 @@ struct unique_fd {
 
 class FirstOrderFilter {
 public:
-  FirstOrderFilter(float x0, float ts, float dt) {
+  FirstOrderFilter(float x0, float ts, float dt, bool initialized = true) {
     k_ = (dt / ts) / (1.0 + dt / ts);
     x_ = x0;
+    initialized_ = initialized;
   }
   inline float update(float x) {
-    x_ = (1. - k_) * x_ + k_ * x;
+    if (initialized_) {
+      x_ = (1. - k_) * x_ + k_ * x;
+    } else {
+      initialized_ = true;
+      x_ = x;
+    }
     return x_;
   }
   inline void reset(float x) { x_ = x; }
@@ -164,12 +172,13 @@ public:
 
 private:
   float x_, k_;
+  bool initialized_;
 };
 
 template<typename T>
 void update_max_atomic(std::atomic<T>& max, T const& value) {
   T prev = max;
-  while(prev < value && !max.compare_exchange_weak(prev, value)) {}
+  while (prev < value && !max.compare_exchange_weak(prev, value)) {}
 }
 
 class LogState {

common/watchdog.cc

@@ -1,3 +1,5 @@
+#include <string>
+
 #include "common/watchdog.h"
 #include "common/util.h"
 

common/window.py

@@ -1,6 +1,6 @@
 import sys
-import pygame  # pylint: disable=import-error
+import pygame
-import cv2  # pylint: disable=import-error
+import cv2
 
 class Window:
   def __init__(self, w, h, caption="window", double=False, halve=False):

docs/CARS.md

@@ -4,7 +4,7 @@
 
 A supported vehicle is one that just works when you install a comma three. All supported cars provide a better experience than any stock system. Supported vehicles reference the US market unless otherwise specified.
 
-# 257 Supported Cars
+# 258 Supported Cars
 
 |Make|Model|Supported Package|ACC|No ACC accel below|No ALC below|Steering Torque|Resume from stop|<a href="##"><img width=2000></a>Hardware Needed<br>&nbsp;|Video|
 |---|---|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|
@@ -81,7 +81,7 @@ A supported vehicle is one that just works when you install a comma three. All s
 |Hyundai|Ioniq 5 (with HDA II) 2022-23[<sup>6</sup>](#footnotes)|Highway Driving Assist II|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai Q connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Ioniq 5 (with HDA II) 2022-23">Buy Here</a></sub></details>||
 |Hyundai|Ioniq 5 (without HDA II) 2022-23[<sup>6</sup>](#footnotes)|Highway Driving Assist|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai K connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Ioniq 5 (without HDA II) 2022-23">Buy Here</a></sub></details>||
 |Hyundai|Ioniq Electric 2019|Smart Cruise Control (SCC)|Stock|0 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai C connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Ioniq Electric 2019">Buy Here</a></sub></details>||
-|Hyundai|Ioniq Electric 2020|All|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai H connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Ioniq Electric 2020">Buy Here</a></sub></details>||
+|Hyundai|Ioniq Electric 2020|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai H connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Ioniq Electric 2020">Buy Here</a></sub></details>||
 |Hyundai|Ioniq Hybrid 2017-19|Smart Cruise Control (SCC)|Stock|0 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai C connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Ioniq Hybrid 2017-19">Buy Here</a></sub></details>||
 |Hyundai|Ioniq Hybrid 2020-22|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai H connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Ioniq Hybrid 2020-22">Buy Here</a></sub></details>||
 |Hyundai|Ioniq Plug-in Hybrid 2019|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai C connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Ioniq Plug-in Hybrid 2019">Buy Here</a></sub></details>||
@@ -124,15 +124,16 @@ A supported vehicle is one that just works when you install a comma three. All s
 |Kia|Niro EV 2023[<sup>6</sup>](#footnotes)|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai A connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Niro EV 2023">Buy Here</a></sub></details>||
 |Kia|Niro Hybrid 2021-22|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai F connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Niro Hybrid 2021-22">Buy Here</a></sub></details>||
 |Kia|Niro Hybrid 2023[<sup>6</sup>](#footnotes)|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai A connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Niro Hybrid 2023">Buy Here</a></sub></details>||
-|Kia|Niro Plug-in Hybrid 2018-19|All|openpilot available[<sup>1</sup>](#footnotes)|10 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai C connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Niro Plug-in Hybrid 2018-19">Buy Here</a></sub></details>||
+|Kia|Niro Plug-in Hybrid 2018-19|All|Stock|10 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai C connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Niro Plug-in Hybrid 2018-19">Buy Here</a></sub></details>||
-|Kia|Niro Plug-in Hybrid 2020|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai D connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Niro Plug-in Hybrid 2020">Buy Here</a></sub></details>||
+|Kia|Niro Plug-in Hybrid 2020|All|Stock|0 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai D connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Niro Plug-in Hybrid 2020">Buy Here</a></sub></details>||
 |Kia|Optima 2017|Advanced Smart Cruise Control|Stock|0 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai B connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Optima 2017">Buy Here</a></sub></details>||
 |Kia|Optima 2019-20|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai G connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Optima 2019-20">Buy Here</a></sub></details>||
 |Kia|Seltos 2021|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai A connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Seltos 2021">Buy Here</a></sub></details>||
 |Kia|Sorento 2018|Advanced Smart Cruise Control|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai C connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Sorento 2018">Buy Here</a></sub></details>|<a href="https://www.youtube.com/watch?v=Fkh3s6WHJz8" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Kia|Sorento 2019|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai E connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Sorento 2019">Buy Here</a></sub></details>|<a href="https://www.youtube.com/watch?v=Fkh3s6WHJz8" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Kia|Sorento 2021-23[<sup>6</sup>](#footnotes)|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai K connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Sorento 2021-23">Buy Here</a></sub></details>||
-|Kia|Sorento Plug-in Hybrid 2022-23[<sup>6</sup>](#footnotes)|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai A connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Sorento Plug-in Hybrid 2022-23">Buy Here</a></sub></details>||
+|Kia|Sorento Hybrid 2023[<sup>6</sup>](#footnotes)|All|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai A connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Sorento Hybrid 2023">Buy Here</a></sub></details>||
+|Kia|Sorento Plug-in Hybrid 2022-23[<sup>6</sup>](#footnotes)|All|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai A connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Sorento Plug-in Hybrid 2022-23">Buy Here</a></sub></details>||
 |Kia|Sportage 2023[<sup>6</sup>](#footnotes)|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai N connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Sportage 2023">Buy Here</a></sub></details>||
 |Kia|Sportage Hybrid 2023[<sup>6</sup>](#footnotes)|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai N connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Sportage Hybrid 2023">Buy Here</a></sub></details>||
 |Kia|Stinger 2018-20|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai C connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma power v2<br>- 1 comma three<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Stinger 2018-20">Buy Here</a></sub></details>|<a href="https://www.youtube.com/watch?v=MJ94qoofYw0" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|

docs/conf.py

@@ -1,5 +1,4 @@
 # type: ignore
-# pylint: skip-file
 
 # Configuration file for the Sphinx documentation builder.
 #
@@ -17,8 +16,8 @@ import os
 import sys
 from os.path import exists
 
-from common.basedir import BASEDIR
+from openpilot.common.basedir import BASEDIR
-from system.version import get_version
+from openpilot.system.version import get_version
 
 sys.path.insert(0, os.path.abspath('.'))
 sys.path.insert(0, os.path.abspath('..'))

docs/docker/Dockerfile

@@ -1,4 +1,6 @@
-FROM ghcr.io/commaai/openpilot-base:latest as openpilot-docs-base
+FROM scons-cache as scons-cache
+
+FROM ghcr.io/commaai/openpilot-base:latest
 
 ENV PYTHONUNBUFFERED 1
 
@@ -11,6 +13,7 @@ WORKDIR ${OPENPILOT_PATH}
 
 COPY SConstruct ${OPENPILOT_PATH}
 
+COPY ./openpilot ${OPENPILOT_PATH}/openpilot
 COPY ./body ${OPENPILOT_PATH}/body
 COPY ./third_party ${OPENPILOT_PATH}/third_party
 COPY ./site_scons ${OPENPILOT_PATH}/site_scons
@@ -28,7 +31,7 @@ COPY ./selfdrive ${OPENPILOT_PATH}/selfdrive
 COPY ./system ${OPENPILOT_PATH}/system
 COPY ./*.md ${OPENPILOT_PATH}/
 
-RUN scons -j$(nproc)
+RUN --mount=type=bind,from=scons-cache,source=/tmp/scons_cache,target=/tmp/scons_cache,rw scons -j$(nproc)
 
 RUN apt update && apt install doxygen -y
 COPY ./docs ${OPENPILOT_PATH}/docs
@@ -37,5 +40,5 @@ WORKDIR ${OPENPILOT_PATH}/docs
 RUN make html
 
 FROM nginx:1.21
-COPY --from=0  /home/batman/openpilot/build/docs/html /usr/share/nginx/html
+COPY --from=1 /home/batman/openpilot/build/docs/html /usr/share/nginx/html
 COPY ./docs/docker/nginx.conf /etc/nginx/conf.d/default.conf

laika_repo

@@ -1 +1 @@
-Subproject commit ef21c612f4b0d68c790b4b1a6d637cd5fce7732e
+Subproject commit a028f6a2c95adf03fa240a9479ad33290969055e

mypy.ini

@@ -1,16 +0,0 @@
-[mypy]
-python_version = 3.11
-plugins = numpy.typing.mypy_plugin
-files = body, common, docs, scripts, selfdrive, site_scons, system, tools
-exclude = ^(cereal/)|(opendbc/)|(panda/)|(laika/)|(laika_repo/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(xx/)
-
-; third-party packages
-ignore_missing_imports = True
-
-; helpful warnings
-warn_redundant_casts = True
-warn_unreachable = True
-warn_unused_ignores = True
-
-; restrict dynamic typing
-warn_return_any = True

opendbc

@@ -1 +1 @@
-Subproject commit a1582f5e28fe0df23b6821c907188be477aac11c
+Subproject commit a30a6775a5013bd42626ae7e36a1d21c9082a333

panda

@@ -1 +1 @@
-Subproject commit 1f475a8affc5270227af18e9c418b5fcd12575b9
+Subproject commit 09cd81752d1d53de4cc1c63f950dc717acc20ad8

poetry.lock

@@ -387,9 +387,7 @@ version = "0.9.14"
 description = "Python API for communicating with the CARLA server."
 optional = false
 python-versions = "*"
-files = [
+files = []
-    {file = "carla-0.9.14-cp311-cp311-linux_x86_64.whl", hash = "sha256:f0a8ce0c760d1fef3577e2ef90e9d468e3d85e65bd6d68b44bce51f0d5a0723a"},
-]
 
 [package.source]
 type = "url"
@@ -732,6 +730,27 @@ mypy = ["contourpy[bokeh,docs]", "docutils-stubs", "mypy (==1.2.0)", "types-Pill
 test = ["Pillow", "contourpy[test-no-images]", "matplotlib"]
 test-no-images = ["pytest", "pytest-cov", "wurlitzer"]
 
+[[package]]
+name = "control"
+version = "0.9.4"
+description = "Python Control Systems Library"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "control-0.9.4-py3-none-any.whl", hash = "sha256:ab68980abd8d35ae5015ffa090865cbbd926deea7e66d0b9a41cfd12577e63ff"},
+    {file = "control-0.9.4.tar.gz", hash = "sha256:0fa57d2216b7ac4e9339c09eab6827660318a641779335864feee940bd19c9ce"},
+]
+
+[package.dependencies]
+matplotlib = "*"
+numpy = "*"
+scipy = ">=1.3"
+
+[package.extras]
+cvxopt = ["cvxopt (>=1.2.0)"]
+slycot = ["slycot (>=0.4.0)"]
+test = ["pytest", "pytest-timeout"]
+
 [[package]]
 name = "coverage"
 version = "7.3.0"
@@ -2178,39 +2197,36 @@ setuptools = "*"
 
 [[package]]
 name = "numpy"
-version = "1.23.5"
+version = "1.25.2"
-description = "NumPy is the fundamental package for array computing with Python."
+description = "Fundamental package for array computing in Python"
 optional = false
-python-versions = ">=3.8"
+python-versions = ">=3.9"
 files = [
-    {file = "numpy-1.23.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c88793f78fca17da0145455f0d7826bcb9f37da4764af27ac945488116efe63"},
+    {file = "numpy-1.25.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db3ccc4e37a6873045580d413fe79b68e47a681af8db2e046f1dacfa11f86eb3"},
-    {file = "numpy-1.23.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e9f4c4e51567b616be64e05d517c79a8a22f3606499941d97bb76f2ca59f982d"},
+    {file = "numpy-1.25.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:90319e4f002795ccfc9050110bbbaa16c944b1c37c0baeea43c5fb881693ae1f"},
-    {file = "numpy-1.23.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7903ba8ab592b82014713c491f6c5d3a1cde5b4a3bf116404e08f5b52f6daf43"},
+    {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfe4a913e29b418d096e696ddd422d8a5d13ffba4ea91f9f60440a3b759b0187"},
-    {file = "numpy-1.23.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e05b1c973a9f858c74367553e236f287e749465f773328c8ef31abe18f691e1"},
+    {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f08f2e037bba04e707eebf4bc934f1972a315c883a9e0ebfa8a7756eabf9e357"},
-    {file = "numpy-1.23.5-cp310-cp310-win32.whl", hash = "sha256:522e26bbf6377e4d76403826ed689c295b0b238f46c28a7251ab94716da0b280"},
+    {file = "numpy-1.25.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bec1e7213c7cb00d67093247f8c4db156fd03075f49876957dca4711306d39c9"},
-    {file = "numpy-1.23.5-cp310-cp310-win_amd64.whl", hash = "sha256:dbee87b469018961d1ad79b1a5d50c0ae850000b639bcb1b694e9981083243b6"},
+    {file = "numpy-1.25.2-cp310-cp310-win32.whl", hash = "sha256:7dc869c0c75988e1c693d0e2d5b26034644399dd929bc049db55395b1379e044"},
-    {file = "numpy-1.23.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ce571367b6dfe60af04e04a1834ca2dc5f46004ac1cc756fb95319f64c095a96"},
+    {file = "numpy-1.25.2-cp310-cp310-win_amd64.whl", hash = "sha256:834b386f2b8210dca38c71a6e0f4fd6922f7d3fcff935dbe3a570945acb1b545"},
-    {file = "numpy-1.23.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56e454c7833e94ec9769fa0f86e6ff8e42ee38ce0ce1fa4cbb747ea7e06d56aa"},
+    {file = "numpy-1.25.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c5462d19336db4560041517dbb7759c21d181a67cb01b36ca109b2ae37d32418"},
-    {file = "numpy-1.23.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5039f55555e1eab31124a5768898c9e22c25a65c1e0037f4d7c495a45778c9f2"},
+    {file = "numpy-1.25.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5652ea24d33585ea39eb6a6a15dac87a1206a692719ff45d53c5282e66d4a8f"},
-    {file = "numpy-1.23.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58f545efd1108e647604a1b5aa809591ccd2540f468a880bedb97247e72db387"},
+    {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2"},
-    {file = "numpy-1.23.5-cp311-cp311-win32.whl", hash = "sha256:b2a9ab7c279c91974f756c84c365a669a887efa287365a8e2c418f8b3ba73fb0"},
+    {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60e7f0f7f6d0eee8364b9a6304c2845b9c491ac706048c7e8cf47b83123b8dbf"},
-    {file = "numpy-1.23.5-cp311-cp311-win_amd64.whl", hash = "sha256:0cbe9848fad08baf71de1a39e12d1b6310f1d5b2d0ea4de051058e6e1076852d"},
+    {file = "numpy-1.25.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bb33d5a1cf360304754913a350edda36d5b8c5331a8237268c48f91253c3a364"},
-    {file = "numpy-1.23.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f063b69b090c9d918f9df0a12116029e274daf0181df392839661c4c7ec9018a"},
+    {file = "numpy-1.25.2-cp311-cp311-win32.whl", hash = "sha256:5883c06bb92f2e6c8181df7b39971a5fb436288db58b5a1c3967702d4278691d"},
-    {file = "numpy-1.23.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0aaee12d8883552fadfc41e96b4c82ee7d794949e2a7c3b3a7201e968c7ecab9"},
+    {file = "numpy-1.25.2-cp311-cp311-win_amd64.whl", hash = "sha256:5c97325a0ba6f9d041feb9390924614b60b99209a71a69c876f71052521d42a4"},
-    {file = "numpy-1.23.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92c8c1e89a1f5028a4c6d9e3ccbe311b6ba53694811269b992c0b224269e2398"},
+    {file = "numpy-1.25.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b79e513d7aac42ae918db3ad1341a015488530d0bb2a6abcbdd10a3a829ccfd3"},
-    {file = "numpy-1.23.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d208a0f8729f3fb790ed18a003f3a57895b989b40ea4dce4717e9cf4af62c6bb"},
+    {file = "numpy-1.25.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb942bfb6f84df5ce05dbf4b46673ffed0d3da59f13635ea9b926af3deb76926"},
-    {file = "numpy-1.23.5-cp38-cp38-win32.whl", hash = "sha256:06005a2ef6014e9956c09ba07654f9837d9e26696a0470e42beedadb78c11b07"},
+    {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e0746410e73384e70d286f93abf2520035250aad8c5714240b0492a7302fdca"},
-    {file = "numpy-1.23.5-cp38-cp38-win_amd64.whl", hash = "sha256:ca51fcfcc5f9354c45f400059e88bc09215fb71a48d3768fb80e357f3b457e1e"},
+    {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7806500e4f5bdd04095e849265e55de20d8cc4b661b038957354327f6d9b295"},
-    {file = "numpy-1.23.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8969bfd28e85c81f3f94eb4a66bc2cf1dbdc5c18efc320af34bffc54d6b1e38f"},
+    {file = "numpy-1.25.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8b77775f4b7df768967a7c8b3567e309f617dd5e99aeb886fa14dc1a0791141f"},
-    {file = "numpy-1.23.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a7ac231a08bb37f852849bbb387a20a57574a97cfc7b6cabb488a4fc8be176de"},
+    {file = "numpy-1.25.2-cp39-cp39-win32.whl", hash = "sha256:2792d23d62ec51e50ce4d4b7d73de8f67a2fd3ea710dcbc8563a51a03fb07b01"},
-    {file = "numpy-1.23.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf837dc63ba5c06dc8797c398db1e223a466c7ece27a1f7b5232ba3466aafe3d"},
+    {file = "numpy-1.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:76b4115d42a7dfc5d485d358728cdd8719be33cc5ec6ec08632a5d6fca2ed380"},
-    {file = "numpy-1.23.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33161613d2269025873025b33e879825ec7b1d831317e68f4f2f0f84ed14c719"},
+    {file = "numpy-1.25.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1a1329e26f46230bf77b02cc19e900db9b52f398d6722ca853349a782d4cff55"},
-    {file = "numpy-1.23.5-cp39-cp39-win32.whl", hash = "sha256:af1da88f6bc3d2338ebbf0e22fe487821ea4d8e89053e25fa59d1d79786e7481"},
+    {file = "numpy-1.25.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c3abc71e8b6edba80a01a52e66d83c5d14433cbcd26a40c329ec7ed09f37901"},
-    {file = "numpy-1.23.5-cp39-cp39-win_amd64.whl", hash = "sha256:09b7847f7e83ca37c6e627682f145856de331049013853f344f37b0c9690e3df"},
+    {file = "numpy-1.25.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1b9735c27cea5d995496f46a8b1cd7b408b3f34b6d50459d9ac8fe3a20cc17bf"},
-    {file = "numpy-1.23.5-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:abdde9f795cf292fb9651ed48185503a2ff29be87770c3b8e2a14b0cd7aa16f8"},
+    {file = "numpy-1.25.2.tar.gz", hash = "sha256:fd608e19c8d7c55021dffd43bfe5492fab8cc105cc8986f813f8c3c048b38760"},
-    {file = "numpy-1.23.5-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9a909a8bae284d46bbfdefbdd4a262ba19d3bc9921b1e76126b1d21c3c34135"},
-    {file = "numpy-1.23.5-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:01dd17cbb340bf0fc23981e52e1d18a9d4050792e8fb8363cecbf066a84b827d"},
-    {file = "numpy-1.23.5.tar.gz", hash = "sha256:1b1766d6f397c18153d40015ddfc79ddb715cabadc04d2d228d4e5a8bc4ded1a"},
 ]
 
 [[package]]
@@ -3108,6 +3124,39 @@ pluggy = ">=0.12,<2.0"
 [package.extras]
 testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
 
+[[package]]
+name = "pytest-cov"
+version = "4.1.0"
+description = "Pytest plugin for measuring coverage."
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "pytest-cov-4.1.0.tar.gz", hash = "sha256:3904b13dfbfec47f003b8e77fd5b589cd11904a21ddf1ab38a64f204d6a10ef6"},
+    {file = "pytest_cov-4.1.0-py3-none-any.whl", hash = "sha256:6ba70b9e97e69fcc3fb45bfeab2d0a138fb65c4d0d6a41ef33983ad114be8c3a"},
+]
+
+[package.dependencies]
+coverage = {version = ">=5.2.1", extras = ["toml"]}
+pytest = ">=4.6"
+
+[package.extras]
+testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"]
+
+[[package]]
+name = "pytest-subtests"
+version = "0.11.0"
+description = "unittest subTest() support and subtests fixture"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "pytest-subtests-0.11.0.tar.gz", hash = "sha256:51865c88457545f51fb72011942f0a3c6901ee9e24cbfb6d1b9dc1348bafbe37"},
+    {file = "pytest_subtests-0.11.0-py3-none-any.whl", hash = "sha256:453389984952eec85ab0ce0c4f026337153df79587048271c7fd0f49119c07e4"},
+]
+
+[package.dependencies]
+attrs = ">=19.2.0"
+pytest = ">=7.0"
+
 [[package]]
 name = "pytest-xdist"
 version = "3.3.1"
@@ -4164,4 +4213,4 @@ multidict = ">=4.0"
 [metadata]
 lock-version = "2.0"
 python-versions = "~3.11"
-content-hash = "849f9d091efd79cb02fffa24846ddb1dd2857cfdce45527093e3bef7d0bc3598"
+content-hash = "320aa5cc075d746403abb872211ebdaf55e9504face8dab8c9b270c71953675d"

pyproject.toml

@@ -4,6 +4,37 @@ addopts = "--ignore=panda/ --ignore=rednose_repo/ --ignore=tinygrad_repo/ --igno
 python_files = "test_*.py"
 timeout = "30"  # you get this long by default
 
+[tool.mypy]
+python_version = "3.11"
+plugins = [
+  "numpy.typing.mypy_plugin",
+]
+exclude = [
+  "body/",
+  "cereal/",
+  "opendbc/",
+  "panda/",
+  "laika/",
+  "laika_repo/",
+  "rednose/",
+  "rednose_repo/",
+  "tinygrad/",
+  "tinygrad_repo/",
+  "third_party/",
+]
+
+# third-party packages
+ignore_missing_imports=true
+
+# helpful warnings
+warn_redundant_casts=true
+warn_unreachable=true
+warn_unused_ignores=true
+
+# restrict dynamic typing
+warn_return_any=true
+
+
 [tool.poetry]
 name = "openpilot"
 version = "0.1.0"
@@ -22,6 +53,7 @@ aiohttp = "*"
 aiortc = "*"
 casadi = "==3.6.3"
 cffi = "*"
+control = "*"
 crcmod = "*"
 cryptography = "*"
 Cython = "*"
@@ -31,7 +63,7 @@ hexdump = "*"
 Jinja2 = "*"
 json-rpc = "*"
 libusb1 = "*"
-numpy = "~1.23"  # pinned for acados
+numpy = "*"
 onnx = ">=1.14.0"
 onnxruntime-gpu = { version = ">=1.15.1", platform = "linux", markers = "platform_machine == 'x86_64'" }
 pillow = "*"
@@ -88,6 +120,8 @@ pycurl = "*"
 pygame = "*"
 pyprof2calltree = "*"
 pytest = "*"
+pytest-cov = "*"
+pytest-subtests = "*"
 pytest-xdist = "*"
 scipy = "*"
 sphinx = "*"

release/files_common

@@ -15,12 +15,14 @@ docs/INTEGRATION.md
 docs/LIMITATIONS.md
 site_scons/site_tools/cython.py
 
+openpilot/__init__.py
+openpilot/**
+
 common/.gitignore
 common/__init__.py
 common/conversions.py
 common/gpio.py
 common/realtime.py
-common/clock.pyx
 common/timeout.py
 common/ffi_wrapper.py
 common/file_helpers.py
@@ -160,6 +162,8 @@ common/clutil.cc
 common/clutil.h
 common/params.h
 common/params.cc
+common/ratekeeper.cc
+common/ratekeeper.h
 common/watchdog.cc
 common/watchdog.h
 
@@ -350,6 +354,7 @@ selfdrive/manager/process.py
 selfdrive/manager/test/__init__.py
 selfdrive/manager/test/test_manager.py
 
+selfdrive/modeld/.gitignore
 selfdrive/modeld/__init__.py
 selfdrive/modeld/SConscript
 selfdrive/modeld/modeld.cc
@@ -360,6 +365,10 @@ selfdrive/modeld/modeld
 selfdrive/modeld/navmodeld
 selfdrive/modeld/dmonitoringmodeld
 
+selfdrive/modeld/models/__init__.py
+selfdrive/modeld/models/*.pxd
+selfdrive/modeld/models/*.pyx
+
 selfdrive/modeld/models/commonmodel.cc
 selfdrive/modeld/models/commonmodel.h
 
@@ -388,6 +397,10 @@ selfdrive/modeld/thneed/thneed_common.cc
 selfdrive/modeld/thneed/thneed_qcom2.cc
 selfdrive/modeld/thneed/serialize.cc
 
+selfdrive/modeld/runners/__init__.py
+selfdrive/modeld/runners/*.pxd
+selfdrive/modeld/runners/*.pyx
+
 selfdrive/modeld/runners/snpemodel.cc
 selfdrive/modeld/runners/snpemodel.h
 selfdrive/modeld/runners/thneedmodel.cc
@@ -565,6 +578,7 @@ opendbc/nissan_x_trail_2017_generated.dbc
 opendbc/nissan_leaf_2018_generated.dbc
 
 opendbc/subaru_global_2017_generated.dbc
+opendbc/subaru_global_2020_hybrid_generated.dbc
 opendbc/subaru_outback_2015_generated.dbc
 opendbc/subaru_outback_2019_generated.dbc
 opendbc/subaru_forester_2017_generated.dbc

scripts/count_cars.py

@@ -2,7 +2,7 @@
 from collections import Counter
 from pprint import pprint
 
-from selfdrive.car.docs import get_all_car_info
+from openpilot.selfdrive.car.docs import get_all_car_info
 
 if __name__ == "__main__":
   cars = get_all_car_info()

scripts/disable-powersave.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-from system.hardware import HARDWARE
+from openpilot.system.hardware import HARDWARE
 
 if __name__ == "__main__":
   HARDWARE.set_power_save(False)

scripts/dump_pll.c

@@ -39,9 +39,9 @@ void hexdump(uint32_t *d, int l) {
 
 int main() {
   int fd = open("/dev/mem", O_RDWR);
-  volatile uint32_t *mb = (uint32_t*)mmap(0,0x1000,PROT_READ | PROT_WRITE,MAP_SHARED,fd,0x06400000);
+  volatile uint32_t *mb = (uint32_t *)mmap(0, 0x1000, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0x06400000);
-  volatile uint32_t *mc = (uint32_t*)mmap(0,0x1000,PROT_READ | PROT_WRITE,MAP_SHARED,fd,0x06480000);
+  volatile uint32_t *mc = (uint32_t *)mmap(0, 0x1000, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0x06480000);
-  volatile uint32_t *md = (uint32_t*)mmap(0,0x1000,PROT_READ | PROT_WRITE,MAP_SHARED,fd,0x09A20000);
+  volatile uint32_t *md = (uint32_t *)mmap(0, 0x1000, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0x09A20000);
   while (1) {
     printf("PLL MODE:%x L_VAL:%x ALPHA:%x USER_CTL:%x CONFIG_CTL:%x CONFIG_CTL_HI:%x STATUS:%x TEST_CTL_LO:%x TEST_CTL_HI:%x\n",
         mb[C0_PLL_MODE/4], mb[C0_PLL_L_VAL/4], mb[C0_PLL_ALPHA/4],

scripts/pyqt_demo.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
-from PyQt5.QtWidgets import QApplication, QLabel  # pylint: disable=no-name-in-module, import-error
+from PyQt5.QtWidgets import QApplication, QLabel
-from selfdrive.ui.qt.python_helpers import set_main_window
+from openpilot.selfdrive.ui.qt.python_helpers import set_main_window
 
 
 if __name__ == "__main__":

scripts/waste.py

@@ -1,23 +1,23 @@
 #!/usr/bin/env python3
 import os
+import time
 import numpy as np
-from common.realtime import sec_since_boot
 from multiprocessing import Process
-from setproctitle import setproctitle  # pylint: disable=no-name-in-module
+from setproctitle import setproctitle
 
 def waste(core):
-  os.sched_setaffinity(0, [core,])  # pylint: disable=no-member
+  os.sched_setaffinity(0, [core,])
 
   m1 = np.zeros((200, 200)) + 0.8
   m2 = np.zeros((200, 200)) + 1.2
 
   i = 1
-  st = sec_since_boot()
+  st = time.monotonic()
   j = 0
   while 1:
     if (i % 100) == 0:
       setproctitle("%3d: %8d" % (core, i))
-      lt = sec_since_boot()
+      lt = time.monotonic()
       print("%3d: %8d %f  %.2f" % (core, i, lt-st, j))
       st = lt
     i += 1

selfdrive/athena/athenad.py

@@ -30,17 +30,17 @@ from websocket import (ABNF, WebSocket, WebSocketException, WebSocketTimeoutExce
 import cereal.messaging as messaging
 from cereal import log
 from cereal.services import service_list
-from common.api import Api
+from openpilot.common.api import Api
-from common.basedir import PERSIST
+from openpilot.common.basedir import PERSIST
-from common.file_helpers import CallbackReader
+from openpilot.common.file_helpers import CallbackReader
-from common.params import Params
+from openpilot.common.params import Params
-from common.realtime import sec_since_boot, set_core_affinity
+from openpilot.common.realtime import set_core_affinity
-from system.hardware import HARDWARE, PC, AGNOS
+from openpilot.system.hardware import HARDWARE, PC, AGNOS
-from system.loggerd.config import ROOT
+from openpilot.system.loggerd.config import ROOT
-from system.loggerd.xattr_cache import getxattr, setxattr
+from openpilot.system.loggerd.xattr_cache import getxattr, setxattr
-from selfdrive.statsd import STATS_DIR
+from openpilot.selfdrive.statsd import STATS_DIR
-from system.swaglog import SWAGLOG_DIR, cloudlog
+from openpilot.system.swaglog import SWAGLOG_DIR, cloudlog
-from system.version import get_commit, get_origin, get_short_branch, get_version
+from openpilot.system.version import get_commit, get_origin, get_short_branch, get_version
 
 
 # TODO: use socket constant when mypy recognizes this as a valid attribute
@@ -552,7 +552,7 @@ def getNetworks():
 
 @dispatcher.add_method
 def takeSnapshot() -> Optional[Union[str, Dict[str, str]]]:
-  from system.camerad.snapshot.snapshot import jpeg_write, snapshot
+  from openpilot.system.camerad.snapshot.snapshot import jpeg_write, snapshot
   ret = snapshot()
   if ret is not None:
     def b64jpeg(x):
@@ -593,10 +593,10 @@ def log_handler(end_event: threading.Event) -> None:
     return
 
   log_files = []
-  last_scan = 0
+  last_scan = 0.
   while not end_event.is_set():
     try:
-      curr_scan = sec_since_boot()
+      curr_scan = time.monotonic()
       if curr_scan - last_scan > 10:
         log_files = get_logs_to_send_sorted()
         last_scan = curr_scan
@@ -652,8 +652,8 @@ def log_handler(end_event: threading.Event) -> None:
 
 def stat_handler(end_event: threading.Event) -> None:
   while not end_event.is_set():
-    last_scan = 0
+    last_scan = 0.
-    curr_scan = sec_since_boot()
+    curr_scan = time.monotonic()
     try:
       if curr_scan - last_scan > 10:
         stat_filenames = list(filter(lambda name: not name.startswith(tempfile.gettempprefix()), os.listdir(STATS_DIR)))
@@ -721,7 +721,7 @@ def ws_proxy_send(ws: WebSocket, local_sock: socket.socket, signal_sock: socket.
 
 
 def ws_recv(ws: WebSocket, end_event: threading.Event) -> None:
-  last_ping = int(sec_since_boot() * 1e9)
+  last_ping = int(time.monotonic() * 1e9)
   while not end_event.is_set():
     try:
       opcode, data = ws.recv_data(control_frame=True)
@@ -730,10 +730,10 @@ def ws_recv(ws: WebSocket, end_event: threading.Event) -> None:
           data = data.decode("utf-8")
         recv_queue.put_nowait(data)
       elif opcode == ABNF.OPCODE_PING:
-        last_ping = int(sec_since_boot() * 1e9)
+        last_ping = int(time.monotonic() * 1e9)
         Params().put("LastAthenaPingTime", str(last_ping))
     except WebSocketTimeoutException:
-      ns_since_last_ping = int(sec_since_boot() * 1e9) - last_ping
+      ns_since_last_ping = int(time.monotonic() * 1e9) - last_ping
       if ns_since_last_ping > RECONNECT_TIMEOUT_S * 1e9:
         cloudlog.exception("athenad.ws_recv.timeout")
         end_event.set()

selfdrive/athena/manage_athenad.py

@@ -3,10 +3,10 @@
 import time
 from multiprocessing import Process
 
-from common.params import Params
+from openpilot.common.params import Params
-from selfdrive.manager.process import launcher
+from openpilot.selfdrive.manager.process import launcher
-from system.swaglog import cloudlog
+from openpilot.system.swaglog import cloudlog
-from system.version import get_version, is_dirty
+from openpilot.system.version import get_version, is_dirty
 
 ATHENA_MGR_PID_PARAM = "AthenadPid"
 

selfdrive/athena/registration.py

@@ -6,13 +6,13 @@ from pathlib import Path
 from typing import Optional
 
 from datetime import datetime, timedelta
-from common.api import api_get
+from openpilot.common.api import api_get
-from common.params import Params
+from openpilot.common.params import Params
-from common.spinner import Spinner
+from openpilot.common.spinner import Spinner
-from common.basedir import PERSIST
+from openpilot.common.basedir import PERSIST
-from selfdrive.controls.lib.alertmanager import set_offroad_alert
+from openpilot.selfdrive.controls.lib.alertmanager import set_offroad_alert
-from system.hardware import HARDWARE, PC
+from openpilot.system.hardware import HARDWARE, PC
-from system.swaglog import cloudlog
+from openpilot.system.swaglog import cloudlog
 
 
 UNREGISTERED_DONGLE_ID = "UnregisteredDevice"

selfdrive/athena/tests/helpers.py

@@ -6,7 +6,7 @@ import time
 from functools import wraps
 from multiprocessing import Process
 
-from common.timeout import Timeout
+from openpilot.common.timeout import Timeout
 
 
 class MockResponse:

selfdrive/athena/tests/test_athenad.py

@@ -18,10 +18,10 @@ from unittest import mock
 from websocket import ABNF
 from websocket._exceptions import WebSocketConnectionClosedException
 
-from system import swaglog
+from openpilot.system import swaglog
-from selfdrive.athena import athenad
+from openpilot.selfdrive.athena import athenad
-from selfdrive.athena.athenad import MAX_RETRY_COUNT, dispatcher
+from openpilot.selfdrive.athena.athenad import MAX_RETRY_COUNT, dispatcher
-from selfdrive.athena.tests.helpers import MockWebsocket, MockParams, MockApi, EchoSocket, with_http_server
+from openpilot.selfdrive.athena.tests.helpers import MockWebsocket, MockParams, MockApi, EchoSocket, with_http_server
 from cereal import messaging
 
 
@@ -351,7 +351,7 @@ class TestAthenadMethods(unittest.TestCase):
     self.assertEqual(athenad.upload_queue.qsize(), 1)
     self.assertDictEqual(asdict(athenad.upload_queue.queue[-1]), asdict(item1))
 
-  @mock.patch('selfdrive.athena.athenad.create_connection')
+  @mock.patch('openpilot.selfdrive.athena.athenad.create_connection')
   def test_startLocalProxy(self, mock_create_connection):
     end_event = threading.Event()
 

selfdrive/athena/tests/test_athenad_ping.py

@@ -6,11 +6,11 @@ import unittest
 from typing import Callable, cast, Optional
 from unittest.mock import MagicMock
 
-from common.params import Params
+from openpilot.common.params import Params
-from common.timeout import Timeout
+from openpilot.common.timeout import Timeout
-from selfdrive.athena import athenad
+from openpilot.selfdrive.athena import athenad
-from selfdrive.manager.helpers import write_onroad_params
+from openpilot.selfdrive.manager.helpers import write_onroad_params
-from system.hardware import TICI
+from openpilot.system.hardware import TICI
 
 
 def wifi_radio(on: bool) -> None:

selfdrive/athena/tests/test_registration.py

@@ -7,9 +7,9 @@ from Crypto.PublicKey import RSA
 from pathlib import Path
 from unittest import mock
 
-from common.params import Params
+from openpilot.common.params import Params
-from selfdrive.athena.registration import register, UNREGISTERED_DONGLE_ID
+from openpilot.selfdrive.athena.registration import register, UNREGISTERED_DONGLE_ID
-from selfdrive.athena.tests.helpers import MockResponse
+from openpilot.selfdrive.athena.tests.helpers import MockResponse
 
 
 class TestRegistration(unittest.TestCase):
@@ -23,7 +23,7 @@ class TestRegistration(unittest.TestCase):
     os.mkdir(os.path.join(self.persist.name, "comma"))
     self.priv_key = Path(os.path.join(self.persist.name, "comma/id_rsa"))
     self.pub_key = Path(os.path.join(self.persist.name, "comma/id_rsa.pub"))
-    self.persist_patcher = mock.patch("selfdrive.athena.registration.PERSIST", self.persist.name)
+    self.persist_patcher = mock.patch("openpilot.selfdrive.athena.registration.PERSIST", self.persist.name)
     self.persist_patcher.start()
 
   def tearDown(self):
@@ -44,7 +44,7 @@ class TestRegistration(unittest.TestCase):
     self.params.put("HardwareSerial", "serial")
     self._generate_keys()
 
-    with mock.patch("selfdrive.athena.registration.api_get", autospec=True) as m:
+    with mock.patch("openpilot.selfdrive.athena.registration.api_get", autospec=True) as m:
       dongle = "DONGLE_ID_123"
       self.params.put("DongleId", dongle)
       self.assertEqual(register(), dongle)
@@ -52,7 +52,7 @@ class TestRegistration(unittest.TestCase):
 
   def test_no_keys(self):
     # missing pubkey
-    with mock.patch("selfdrive.athena.registration.api_get", autospec=True) as m:
+    with mock.patch("openpilot.selfdrive.athena.registration.api_get", autospec=True) as m:
       dongle = register()
       self.assertEqual(m.call_count, 0)
       self.assertEqual(dongle, UNREGISTERED_DONGLE_ID)
@@ -61,7 +61,7 @@ class TestRegistration(unittest.TestCase):
   def test_missing_cache(self):
     # keys exist but no dongle id
     self._generate_keys()
-    with mock.patch("selfdrive.athena.registration.api_get", autospec=True) as m:
+    with mock.patch("openpilot.selfdrive.athena.registration.api_get", autospec=True) as m:
       dongle = "DONGLE_ID_123"
       m.return_value = MockResponse(json.dumps({'dongle_id': dongle}), 200)
       self.assertEqual(register(), dongle)
@@ -75,7 +75,7 @@ class TestRegistration(unittest.TestCase):
   def test_unregistered(self):
     # keys exist, but unregistered
     self._generate_keys()
-    with mock.patch("selfdrive.athena.registration.api_get", autospec=True) as m:
+    with mock.patch("openpilot.selfdrive.athena.registration.api_get", autospec=True) as m:
       m.return_value = MockResponse(None, 402)
       dongle = register()
       self.assertEqual(m.call_count, 1)

selfdrive/boardd/SConscript

@@ -7,5 +7,5 @@ env.Program('boardd', ['main.cc', 'boardd.cc'], LIBS=[panda] + libs)
 env.Library('libcan_list_to_can_capnp', ['can_list_to_can_capnp.cc'])
 
 envCython.Program('boardd_api_impl.so', 'boardd_api_impl.pyx', LIBS=["can_list_to_can_capnp", 'capnp', 'kj'] + envCython["LIBS"])
-if GetOption('test'):
+if GetOption('extras'):
   env.Program('tests/test_boardd_usbprotocol', ['tests/test_boardd_usbprotocol.cc'], LIBS=[panda] + libs)

selfdrive/boardd/boardd.cc

@@ -18,11 +18,13 @@
 #include <cstdio>
 #include <cstdlib>
 #include <future>
+#include <memory>
 #include <thread>
 
 #include "cereal/gen/cpp/car.capnp.h"
 #include "cereal/messaging/messaging.h"
 #include "common/params.h"
+#include "common/ratekeeper.h"
 #include "common/swaglog.h"
 #include "common/timing.h"
 #include "common/util.h"
@@ -248,8 +250,7 @@ void can_recv_thread(std::vector<Panda *> pandas) {
   PubMaster pm({"can"});
 
   // run at 100Hz
-  const uint64_t dt = 10000000ULL;
+  RateKeeper rk("boardd_can_recv", 100);
-  uint64_t next_frame_time = nanos_since_boot() + dt;
   std::vector<can_frame> raw_can_data;
 
   while (!do_exit && check_all_connected(pandas)) {
@@ -271,18 +272,7 @@ void can_recv_thread(std::vector<Panda *> pandas) {
     }
     pm.send("can", msg);
 
-    uint64_t cur_time = nanos_since_boot();
+    rk.keepTime();
-    int64_t remaining = next_frame_time - cur_time;
-    if (remaining > 0) {
-      std::this_thread::sleep_for(std::chrono::nanoseconds(remaining));
-    } else {
-      if (ignition) {
-        LOGW("missed cycles (%lu) %lld", (unsigned long)(-1*remaining/dt), (long long)remaining);
-      }
-      next_frame_time = cur_time;
-    }
-
-    next_frame_time += dt;
   }
 }
 
@@ -483,14 +473,16 @@ void panda_state_thread(std::vector<Panda *> pandas, bool spoofing_started) {
   LOGD("start panda state thread");
 
   // run at 2hz
-  while (!do_exit && check_all_connected(pandas)) {
+  RateKeeper rk("panda_state_thread", 2);
-    uint64_t start_time = nanos_since_boot();
 
+  while (!do_exit && check_all_connected(pandas)) {
     // send out peripheralState
     send_peripheral_state(&pm, peripheral_panda);
     auto ignition_opt = send_panda_states(&pm, pandas, spoofing_started);
 
     if (!ignition_opt) {
+      LOGE("Failed to get ignition_opt");
+      rk.keepTime();
       continue;
     }
 
@@ -543,8 +535,7 @@ void panda_state_thread(std::vector<Panda *> pandas, bool spoofing_started) {
       panda->send_heartbeat(engaged);
     }
 
-    uint64_t dt = nanos_since_boot() - start_time;
+    rk.keepTime();
-    util::sleep_for(500 - dt / 1000000ULL);
   }
 }
 

selfdrive/boardd/boardd.h

@@ -1,5 +1,8 @@
 #pragma once
 
+#include <string>
+#include <vector>
+
 #include "selfdrive/boardd/panda.h"
 
 bool safety_setter_thread(std::vector<Panda *> pandas);

selfdrive/boardd/boardd.py

@@ -1,7 +1,5 @@
-# pylint: skip-file
-
 # Cython, now uses scons to build
-from selfdrive.boardd.boardd_api_impl import can_list_to_can_capnp
+from openpilot.selfdrive.boardd.boardd_api_impl import can_list_to_can_capnp
 assert can_list_to_can_capnp
 
 def can_capnp_to_can_list(can, src_filter=None):

selfdrive/boardd/can_list_to_can_capnp.cc

@@ -1,5 +1,5 @@
 #include "cereal/messaging/messaging.h"
-#include "panda.h"
+#include "selfdrive/boardd/panda.h"
 
 void can_list_to_can_capnp_cpp(const std::vector<can_frame> &can_list, std::string &out, bool sendCan, bool valid) {
   MessageBuilder msg;

selfdrive/boardd/panda.cc

@@ -4,6 +4,7 @@
 
 #include <cassert>
 #include <stdexcept>
+#include <vector>
 
 #include "cereal/messaging/messaging.h"
 #include "common/swaglog.h"

selfdrive/boardd/panda.h

@@ -6,6 +6,7 @@
 #include <list>
 #include <memory>
 #include <optional>
+#include <string>
 #include <vector>
 
 #include "cereal/gen/cpp/car.capnp.h"

selfdrive/boardd/panda_comms.cc

@@ -199,7 +199,7 @@ int PandaUsbHandle::bulk_write(unsigned char endpoint, unsigned char* data, int
     } else if (err != 0 || length != transferred) {
       handle_usb_issue(err, __func__);
     }
-  } while(err != 0 && connected);
+  } while (err != 0 && connected);
 
   return transferred;
 }
@@ -226,7 +226,7 @@ int PandaUsbHandle::bulk_read(unsigned char endpoint, unsigned char* data, int l
       handle_usb_issue(err, __func__);
     }
 
-  } while(err != 0 && connected);
+  } while (err != 0 && connected);
 
   return transferred;
 }

selfdrive/boardd/panda_comms.h

@@ -20,8 +20,8 @@
 // comms base class
 class PandaCommsHandle {
 public:
-  PandaCommsHandle(std::string serial) {};
+  PandaCommsHandle(std::string serial) {}
-  virtual ~PandaCommsHandle() {};
+  virtual ~PandaCommsHandle() {}
   virtual void cleanup() = 0;
 
   std::string hw_serial;

selfdrive/boardd/pandad.py

@@ -9,11 +9,11 @@ from typing import List, NoReturn
 from functools import cmp_to_key
 
 from panda import Panda, PandaDFU, PandaProtocolMismatch, FW_PATH
-from common.basedir import BASEDIR
+from openpilot.common.basedir import BASEDIR
-from common.params import Params
+from openpilot.common.params import Params
-from selfdrive.boardd.set_time import set_time
+from openpilot.selfdrive.boardd.set_time import set_time
-from system.hardware import HARDWARE
+from openpilot.system.hardware import HARDWARE
-from system.swaglog import cloudlog
+from openpilot.system.swaglog import cloudlog
 
 
 def get_expected_signature(panda: Panda) -> bytes:

selfdrive/boardd/set_time.py

@@ -3,7 +3,7 @@ import os
 import datetime
 from panda import Panda
 
-from common.time import MIN_DATE
+from openpilot.common.time import MIN_DATE
 
 def set_time(logger):
   sys_time = datetime.datetime.today()

selfdrive/boardd/spi.cc

@@ -43,7 +43,7 @@ public:
   LockEx(int fd, std::recursive_mutex &m) : fd(fd), m(m) {
     m.lock();
     flock(fd, LOCK_EX);
-  };
+  }
 
   ~LockEx() {
     flock(fd, LOCK_UN);

selfdrive/boardd/tests/test_boardd_loopback.py

@@ -9,12 +9,12 @@ from pprint import pprint
 
 import cereal.messaging as messaging
 from cereal import car, log
-from common.params import Params
+from openpilot.common.params import Params
-from common.timeout import Timeout
+from openpilot.common.timeout import Timeout
-from selfdrive.boardd.boardd import can_list_to_can_capnp
+from openpilot.selfdrive.boardd.boardd import can_list_to_can_capnp
-from selfdrive.car import make_can_msg
+from openpilot.selfdrive.car import make_can_msg
-from system.hardware import TICI
+from openpilot.system.hardware import TICI
-from selfdrive.test.helpers import phone_only, with_processes
+from openpilot.selfdrive.test.helpers import phone_only, with_processes
 
 
 class TestBoardd(unittest.TestCase):

selfdrive/boardd/tests/test_boardd_usbprotocol.cc

@@ -82,7 +82,7 @@ void PandaTest::test_can_recv(uint32_t rx_chunk_size) {
       this->receive_buffer_size = 0;
       uint32_t pos = 0;
 
-      while(pos < size) {
+      while (pos < size) {
         uint32_t chunk_size = std::min(rx_chunk_size, size - pos);
         memcpy(&this->receive_buffer[this->receive_buffer_size], &data[pos], chunk_size);
         this->receive_buffer_size += chunk_size;

selfdrive/boardd/tests/test_pandad.py

@@ -5,13 +5,13 @@ import unittest
 
 import cereal.messaging as messaging
 from cereal import log
-from common.gpio import gpio_set, gpio_init
+from openpilot.common.gpio import gpio_set, gpio_init
-from common.params import Params
+from openpilot.common.params import Params
 from panda import Panda, PandaDFU, PandaProtocolMismatch
-from selfdrive.test.helpers import phone_only
+from openpilot.selfdrive.test.helpers import phone_only
-from selfdrive.manager.process_config import managed_processes
+from openpilot.selfdrive.manager.process_config import managed_processes
-from system.hardware import HARDWARE
+from openpilot.system.hardware import HARDWARE
-from system.hardware.tici.pins import GPIO
+from openpilot.system.hardware.tici.pins import GPIO
 
 HERE = os.path.dirname(os.path.realpath(__file__))
 

selfdrive/car/__init__.py

@@ -5,7 +5,7 @@ from typing import Dict, Optional
 import capnp
 
 from cereal import car
-from common.numpy_fast import clip, interp
+from openpilot.common.numpy_fast import clip, interp
 
 
 # kg of standard extra cargo to count for drive, gas, etc...
@@ -40,9 +40,8 @@ def gen_empty_fingerprint():
   return {i: {} for i in range(0, 8)}
 
 
-# FIXME: hardcoding honda civic 2016 touring params so they can be used to
+# these params were derived for the Civic and used to calculate params for other cars
-# scale unknown params for other cars
+class VehicleDynamicsParams:
-class CivicParams:
   MASS = 1326. + STD_CARGO_KG
   WHEELBASE = 2.70
   CENTER_TO_FRONT = WHEELBASE * 0.4
@@ -55,18 +54,18 @@ class CivicParams:
 # TODO: get actual value, for now starting with reasonable value for
 # civic and scaling by mass and wheelbase
 def scale_rot_inertia(mass, wheelbase):
-  return CivicParams.ROTATIONAL_INERTIA * mass * wheelbase ** 2 / (CivicParams.MASS * CivicParams.WHEELBASE ** 2)
+  return VehicleDynamicsParams.ROTATIONAL_INERTIA * mass * wheelbase ** 2 / (VehicleDynamicsParams.MASS * VehicleDynamicsParams.WHEELBASE ** 2)
 
 
 # TODO: start from empirically derived lateral slip stiffness for the civic and scale by
 # mass and CG position, so all cars will have approximately similar dyn behaviors
 def scale_tire_stiffness(mass, wheelbase, center_to_front, tire_stiffness_factor):
   center_to_rear = wheelbase - center_to_front
-  tire_stiffness_front = (CivicParams.TIRE_STIFFNESS_FRONT * tire_stiffness_factor) * mass / CivicParams.MASS * \
+  tire_stiffness_front = (VehicleDynamicsParams.TIRE_STIFFNESS_FRONT * tire_stiffness_factor) * mass / VehicleDynamicsParams.MASS * \
-                         (center_to_rear / wheelbase) / (CivicParams.CENTER_TO_REAR / CivicParams.WHEELBASE)
+                         (center_to_rear / wheelbase) / (VehicleDynamicsParams.CENTER_TO_REAR / VehicleDynamicsParams.WHEELBASE)
 
-  tire_stiffness_rear = (CivicParams.TIRE_STIFFNESS_REAR * tire_stiffness_factor) * mass / CivicParams.MASS * \
+  tire_stiffness_rear = (VehicleDynamicsParams.TIRE_STIFFNESS_REAR * tire_stiffness_factor) * mass / VehicleDynamicsParams.MASS * \
-                        (center_to_front / wheelbase) / (CivicParams.CENTER_TO_FRONT / CivicParams.WHEELBASE)
+                        (center_to_front / wheelbase) / (VehicleDynamicsParams.CENTER_TO_FRONT / VehicleDynamicsParams.WHEELBASE)
 
   return tire_stiffness_front, tire_stiffness_rear
 
@@ -233,4 +232,4 @@ class CanSignalRateCalculator:
     self.previous_counter = current_counter
     self.previous_value = current_value
 
-    return self.rate
+    return self.rate