Merge remote-tracking branch 'upstream/master' into gm-bsm

.github/labeler.yaml

@@ -1,69 +1,79 @@
 CI / testing:
-  - all:
+  - changed-files:
-    - changed-files: ['.github/**', '**/test_*', 'Jenkinsfile']
+    - any-glob-to-all-files: "{.github/**,**/test_*,Jenkinsfile}"
 
 car:
-  - all:
+  - changed-files:
-    - changed-files: ['selfdrive/car/**']
+    - any-glob-to-all-files: 'selfdrive/car/**'
 
 body:
-  - all:
+  - changed-files:
-    - changed-files: ['selfdrive/car/body/*']
+    - any-glob-to-all-files: 'selfdrive/car/body/*'
+
 chrysler:
-  - all:
+  - changed-files:
-    - changed-files: ['selfdrive/car/chrysler/*']
+    - any-glob-to-all-files: 'selfdrive/car/chrysler/*'
+
 ford:
-  - all:
+  - changed-files:
-    - changed-files: ['selfdrive/car/ford/*']
+    - any-glob-to-all-files: 'selfdrive/car/ford/*'
+
 gm:
-  - all:
+  - changed-files:
-    - changed-files: ['selfdrive/car/gm/*']
+    - any-glob-to-all-files: 'selfdrive/car/gm/*'
+
 honda:
-  - all:
+  - changed-files:
-    - changed-files: ['selfdrive/car/honda/*']
+    - any-glob-to-all-files: 'selfdrive/car/honda/*'
+
 hyundai:
-  - all:
+  - changed-files:
-    - changed-files: ['selfdrive/car/hyundai/*']
+    - any-glob-to-all-files: 'selfdrive/car/hyundai/*'
+
 mazda:
-  - all:
+  - changed-files:
-    - changed-files: ['selfdrive/car/mazda/*']
+    - any-glob-to-all-files: 'selfdrive/car/mazda/*'
+
 nissan:
-  - all:
+  - changed-files:
-    - changed-files: ['selfdrive/car/nissan/*']
+    - any-glob-to-all-files: 'selfdrive/car/nissan/*'
+
 subaru:
-  - all:
+  - changed-files:
-    - changed-files: ['selfdrive/car/subaru/*']
+    - any-glob-to-all-files: 'selfdrive/car/subaru/*'
+
 tesla:
-  - all:
+  - changed-files:
-    - changed-files: ['selfdrive/car/tesla/*']
+    - any-glob-to-all-files: 'selfdrive/car/telsa/*'
+
 toyota:
-  - all:
+  - changed-files:
-    - changed-files: ['selfdrive/car/toyota/*']
+    - any-glob-to-all-files: 'selfdrive/car/toyota/*'
+
 volkswagen:
-  - all:
+  - changed-files:
-    - changed-files: ['selfdrive/car/volkswagen/*']
+    - any-glob-to-all-files: 'selfdrive/car/volkswagen/*'
 
 fingerprint:
-  - all:
+  - changed-files:
-    - changed-files: ['selfdrive/car/*/fingerprints.py']
+    - any-glob-to-all-files: 'selfdrive/car/*/fingerprints.py'
 
 simulation:
-  - all:
+  - changed-files:
-    - changed-files: ['tools/sim/**']
+    - any-glob-to-all-files: 'tools/sim/**'
+
 ui:
-  - all:
+  - changed-files:
-    - changed-files: ['selfdrive/ui/**']
+    - any-glob-to-all-files: 'selfdrive/ui/**'
+
 tools:
-  - all:
+  - changed-files:
-    - changed-files: ['tools/**']
+    - any-glob-to-all-files: 'tools/**'
 
 multilanguage:
-  - all:
+  - changed-files:
-    - changed-files: ['selfdrive/ui/translations/**']
+    - any-glob-to-all-files: 'selfdrive/ui/translations/**'
 
 research:
-  - all:
+  - changed-files:
-    - changed-files: [
+    - any-glob-to-all-files: "{selfdrive/modeld/models/**,selfdrive/test/process_replay/model_replay_ref_commit}"
-        'selfdrive/modeld/models/**',
-        'selfdrive/test/process_replay/model_replay_ref_commit',
-      ]

.github/workflows/auto_pr_review.yaml

@@ -1,6 +1,7 @@
 name: "PR review"
 on:
   pull_request_target:
+    types: [opened, reopened, synchronize, edited, edited]
 
 jobs:
   labeler:
@@ -13,7 +14,7 @@ jobs:
     - uses: actions/checkout@v4
       with:
         submodules: false
-    - uses: actions/labeler@v5.0.0-alpha.1
+    - uses: actions/labeler@v5.0.0
       with:
         dot: true
         configuration-path: .github/labeler.yaml
@@ -32,3 +33,168 @@ jobs:
         change-to: ${{ github.base_ref }}
         already-exists-action: close_this
         already-exists-comment: "Your PR should be made against the `master` branch"
+
+  check-pr-template:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      issues: write
+      pull-requests: write
+      actions: read
+    if: github.event.pull_request.head.repo.full_name != 'commaai/openpilot'
+    steps:
+      - uses: actions/github-script@v7
+        with:
+          script: |
+            // Comment to add to the PR if no template has been used
+            const NO_TEMPLATE_MESSAGE =
+              "It looks like you didn't use one of the Pull Request templates. Please check [the contributing docs](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md). \
+            Also make sure that you didn't modify any of the checkboxes or headings within the template.";
+            // body data for future requests
+            const body_data = {
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+            };
+
+            // Utility function to extract all headings
+            const extractHeadings = (markdown) => {
+              const headingRegex = /^(#{1,6})\s+(.+)$/gm;
+              const boldTextRegex = /^(?:\*\*|__)(.+?)(?:\*\*|__)\s*$/gm;
+              const headings = [];
+              let headingMatch;
+              while ((headingMatch = headingRegex.exec(markdown))) {
+                headings.push(headingMatch[2].trim());
+              }
+              let boldMatch;
+              while ((boldMatch = boldTextRegex.exec(markdown))) {
+                headings.push(boldMatch[1].trim());
+              }
+              return headings;
+            };
+
+            // Utility function to extract all check box descriptions
+            const extractCheckBoxTexts = (markdown) => {
+              const checkboxRegex = /^\s*-\s*\[( |x)\]\s+(.+)$/gm;
+              const checkboxes = [];
+              let match;
+              while ((match = checkboxRegex.exec(markdown))) {
+                checkboxes.push(match[2].trim());
+              }
+              return checkboxes;
+            };
+
+            // Utility function to check if a list is a subset of another list
+            isSubset = (subset, superset) => {
+              return subset.every((item) => superset.includes(item));
+            };
+
+            // Get filenames of all currently checked-in PR templates
+            const template_contents = await github.rest.repos.getContent({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              path: ".github/PULL_REQUEST_TEMPLATE",
+            });
+            var template_filenames = [];
+            for (const content of template_contents.data) {
+              template_filenames.push(content.path);
+            }
+            console.debug("Received template filenames: " + template_filenames);
+            // Retrieve templates
+            var templates = [];
+            for (const template_filename of template_filenames) {
+              const template_response = await github.rest.repos.getContent({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                path: template_filename,
+              });
+              // Convert Base64 content back
+              const decoded_template = atob(template_response.data.content);
+              const headings = extractHeadings(decoded_template);
+              const checkboxes = extractCheckBoxTexts(decoded_template);
+              if (!headings.length && !checkboxes.length) {
+                console.warn(
+                  "Invalid template! Contains neither headings nor checkboxes, ignoring it: \n" +
+                    decoded_template
+                );
+              } else {
+                templates.push({ headings: headings, checkboxes: checkboxes });
+              }
+            }
+            // Retrieve the PR Body
+            const pull_request = await github.rest.issues.get({
+              ...body_data,
+            });
+            const pull_request_text = pull_request.data.body;
+            console.debug("Received Pull Request body: \n" + pull_request_text);
+
+            /* Check if the PR Body matches one of the templates
+            A template is defined by all headings and checkboxes it contains
+            We extract all Headings and Checkboxes from the PR text and check if any of the templates is a subset of that
+            */
+            const pr_headings = extractHeadings(pull_request_text);
+            const pr_checkboxes = extractCheckBoxTexts(pull_request_text);
+            console.debug("Found Headings in PR body:\n" + pr_headings);
+            console.debug("Found Checkboxes in PR body:\n" + pr_checkboxes);
+            var template_found = false;
+            // Iterate over each template to check if it applies
+            for (const template of templates) {
+              console.log(
+                "Checking for headings: [" +
+                  template.headings +
+                  "] and checkboxes: [" +
+                  template.checkboxes + "]"
+              );
+              if (
+                isSubset(template.checkboxes, pr_checkboxes) &&
+                isSubset(template.headings, pr_headings)
+              ) {
+                console.debug("Found matching template!");
+                template_found = true;
+              }
+            }
+
+            // List comments from previous runs
+            var existing_comments = [];
+            const comments = await github.rest.issues.listComments({
+              ...body_data,
+            });
+            for (const comment of comments.data) {
+              if (comment.body === NO_TEMPLATE_MESSAGE) {
+                existing_comments.push(comment);
+              }
+            }
+
+            // Add a comment to the PR that it is not using a the template (but only if this comment does not exist already)
+            if (!template_found) {
+              var comment_already_sent = false;
+
+              // Add an 'in-bot-review' label since this PR doesn't have the template
+              github.rest.issues.addLabels({
+                ...body_data,
+                labels: ["in-bot-review"],
+              });
+
+              if (existing_comments.length < 1) {
+                github.rest.issues.createComment({
+                  ...body_data,
+                  body: NO_TEMPLATE_MESSAGE,
+                });
+              }
+            } else {
+              // If template has been found, delete any old comment about missing template
+              for (const existing_comment of existing_comments) {
+                github.rest.issues.deleteComment({
+                  ...body_data,
+                  comment_id: existing_comment.id,
+                });
+              }
+              // Remove the 'in-bot-review' label after the review is done and the PR has passed
+              github.rest.issues.removeLabel({
+                ...body_data,
+                name: "in-bot-review",
+              }).catch((error) => {
+                console.log("Label 'in-bot-review' not found, ignoring");
+              });
+            }
+

.github/workflows/badges.yaml

@@ -14,6 +14,8 @@ jobs:
     name: create badges
     runs-on: ubuntu-20.04
     if: github.repository == 'commaai/openpilot'
+    permissions:
+      contents: write
     steps:
     - uses: actions/checkout@v4
       with:
@@ -23,6 +25,8 @@ jobs:
       run: |
         ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/ui/translations/create_badges.py"
 
+        rm .gitattributes
+
         git checkout --orphan badges
         git rm -rf --cached .
         git config user.email "badge-researcher@comma.ai"

.github/workflows/prebuilt.yaml

@@ -15,14 +15,19 @@ jobs:
     if: github.repository == 'commaai/openpilot'
     env:
       PUSH_IMAGE: true
+    permissions:
+      checks: read
+      contents: read
+      packages: write
     steps:
     - name: Wait for green check mark
       if: ${{ github.event_name != 'workflow_dispatch' }}
-      uses: lewagon/wait-on-check-action@e2558238c09778af25867eb5de5a3ce4bbae3dcd
+      uses: lewagon/wait-on-check-action@595dabb3acf442d47e29c9ec9ba44db0c6bdd18f
       with:
         ref: master
         wait-interval: 30
         running-workflow-name: 'build prebuilt'
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
         check-regexp: ^((?!.*(build master-ci).*).)*$
     - uses: actions/checkout@v4
       with:

.github/workflows/release.yaml

@@ -14,6 +14,9 @@ jobs:
       image: ghcr.io/commaai/openpilot-base:latest
     runs-on: ubuntu-20.04
     if: github.repository == 'commaai/openpilot'
+    permissions:
+      checks: read
+      contents: write
     steps:
     - name: Install wait-on-check-action dependencies
       run: |
@@ -21,11 +24,12 @@ jobs:
         sudo apt-get install -y libyaml-dev
     - name: Wait for green check mark
       if: ${{ github.event_name != 'workflow_dispatch' }}
-      uses: lewagon/wait-on-check-action@e2558238c09778af25867eb5de5a3ce4bbae3dcd
+      uses: lewagon/wait-on-check-action@595dabb3acf442d47e29c9ec9ba44db0c6bdd18f
       with:
         ref: master
         wait-interval: 30
         running-workflow-name: 'build master-ci'
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
         check-regexp: ^((?!.*(build prebuilt).*).)*$
     - uses: actions/checkout@v4
       with:

.github/workflows/selfdrive_tests.yaml

@@ -20,11 +20,11 @@ env:
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
   BUILD: selfdrive/test/docker_build.sh base
 
-  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
 
   BUILD_CL: selfdrive/test/docker_build.sh cl
 
-  RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/bash -c
+  RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/bash -c
 
   PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0
 
@@ -51,7 +51,7 @@ jobs:
       timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
       run: |
         cd $STRIPPED_DIR
-        ${{ env.RUN }} "CI=1 python selfdrive/manager/build.py"
+        ${{ env.RUN }} "python selfdrive/manager/build.py"
     - name: Run tests
       timeout-minutes: 3
       run: |
@@ -163,12 +163,16 @@ jobs:
 
   unit_tests:
     name: unit tests
-    runs-on: ubuntu-20.04
+    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
+                   ((github.event_name != 'pull_request') || 
+                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'buildjet-8vcpu-ubuntu-2004' || 'ubuntu-20.04' }}
     steps:
     - uses: actions/checkout@v4
       with:
         submodules: true
     - uses: ./.github/workflows/setup-with-retry
+      with:
+        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
     - name: Build openpilot
       timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
       run: ${{ env.RUN }} "scons -j$(nproc)"
@@ -180,8 +184,7 @@ jobs:
                         $PYTEST --timeout 30 -m 'not slow' && \
                         ./selfdrive/ui/tests/create_test_translations.sh && \
                         QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
-                        ./selfdrive/ui/tests/test_translations.py && \
+                        ./selfdrive/ui/tests/test_translations.py"
-                        ./system/camerad/test/ae_gray_test"
     - name: "Upload coverage to Codecov"
       uses: codecov/codecov-action@v3
       with:
@@ -213,7 +216,7 @@ jobs:
     - name: Run replay
       timeout-minutes: 30
       run: |
-        ${{ env.RUN }} "CI=1 coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
+        ${{ env.RUN }} "coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
                         chmod -R 777 /tmp/comma_download_cache && \
                         coverage combine && \
                         coverage xml"
@@ -230,7 +233,7 @@ jobs:
     - name: Upload reference logs
       if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
       run: |
-        ${{ env.RUN }} "unset PYTHONWARNINGS && CI=1 AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
+        ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
     - name: "Upload coverage to Codecov"
       uses: codecov/codecov-action@v3
       with:
@@ -286,7 +289,7 @@ jobs:
       timeout-minutes: 4
       run: |
         ${{ env.RUN_CL }} "unset PYTHONWARNINGS && \
-                           ONNXCPU=1 CI=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \
+                           ONNXCPU=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \
                            coverage combine && \
                            coverage xml"
     - name: Run unit tests

.github/workflows/stale.yaml

@@ -20,7 +20,7 @@ jobs:
           stale-pr-message: 'This PR has had no activity for ${{ env.DAYS_BEFORE_PR_STALE }} days. It will be automatically closed in ${{ env.DAYS_BEFORE_PR_CLOSE }} days if there is no activity.'
           close-pr-message: 'This PR has been automatically closed due to inactivity. Feel free to re-open once activity resumes.'
           delete-branch: ${{ github.event.pull_request.head.repo.full_name == 'commaai/openpilot' }} # only delete branches on the main repo
-          exempt-pr-labels: "ignore stale,needs testing" # if wip or it needs testing from the community, don't mark as stale
+          exempt-pr-labels: "ignore stale,needs testing,car port" # if wip or it needs testing from the community, don't mark as stale
           days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE }}
           days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }}
 

.github/workflows/tools_tests.yaml

@@ -85,7 +85,7 @@ jobs:
         devcontainer up --workspace-folder .
     - name: Test environment
       run: |
-        devcontainer exec --workspace-folder . scons -j$(nproc)
+        devcontainer exec --workspace-folder . scons -j$(nproc) cereal/ common/
         devcontainer exec --workspace-folder . pip install pip-install-test
         devcontainer exec --workspace-folder . touch /home/batman/.comma/auth.json
         devcontainer exec --workspace-folder . sudo touch /root/test.txt

.pre-commit-config.yaml

@@ -39,10 +39,12 @@ repos:
         entry: mypy
         language: system
         types: [python]
-        args: ['--explicit-package-bases', '--local-partial-types']
+        args:
+        - --local-partial-types
+        - --explicit-package-bases
         exclude: '^(third_party/)|(cereal/)|(opendbc/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)|(xx/)'
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.6
+    rev: v0.1.13
     hooks:
     -   id: ruff
         exclude: '^(third_party/)|(cereal/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)'
@@ -87,7 +89,7 @@ repos:
         args:
         - --lock
 -   repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.27.2
+    rev: 0.27.3
     hooks:
     -   id: check-github-workflows
 # -   repo: local

Jenkinsfile

@@ -110,18 +110,20 @@ def pcStage(String stageName, Closure body) {
       return docker.build("openpilot-base:build-${env.GIT_COMMIT}", "-f Dockerfile.openpilot_base .")
     }
 
-    openpilot_base.inside(dockerArgs) {
+    lock(resource: "", label: 'pc', inversePrecedence: true, quantity: 1) {
-      timeout(time: 20, unit: 'MINUTES') {
+      openpilot_base.inside(dockerArgs) {
-        try {
+        timeout(time: 20, unit: 'MINUTES') {
-          retryWithDelay (3, 15) {
+          try {
-            sh "git config --global --add safe.directory '*'"
+            retryWithDelay (3, 15) {
-            sh "git submodule update --init --recursive"
+              sh "git config --global --add safe.directory '*'"
-            sh "git lfs pull"
+              sh "git submodule update --init --recursive"
-          }
+              sh "git lfs pull"
-          body()
+            }
-        } finally {
+            body()
-            sh "rm -rf ${env.WORKSPACE}/* || true"
+          } finally {
-            sh "rm -rf .* || true"
+              sh "rm -rf ${env.WORKSPACE}/* || true"
+              sh "rm -rf .* || true"
+            }
           }
         }
       }
@@ -241,7 +243,7 @@ node {
       'replay': {
         deviceStage("tici", "tici-replay", ["UNSAFE=1"], [
           ["build", "cd selfdrive/manager && ./build.py"],
-          ["model replay", "selfdrive/test/process_replay/model_replay.py", ["tinygrad/", "selfdrive/modeld/"]],
+          ["model replay", "selfdrive/test/process_replay/model_replay.py"],
         ])
       },
       'tizi': {

RELEASES.md

@@ -7,8 +7,8 @@ Version 0.9.6 (20XX-XX-XX)
 * comma body streaming and controls over WebRTC
 * Hyundai Staria 2023 support thanks to sunnyhaibin!
 * Kia Niro Plug-in Hybrid 2022 support thanks to sunnyhaibin!
-* Toyota RAV4 2023 support
+* Toyota RAV4 2023-24 support
-* Toyota RAV4 Hybrid 2023 support
+* Toyota RAV4 Hybrid 2023-24 support
 
 Version 0.9.5 (2023-11-17)
 ========================

SConstruct

@@ -9,11 +9,16 @@ import SCons.Errors
 
 SCons.Warnings.warningAsException(True)
 
+# pending upstream fix - https://github.com/SCons/scons/issues/4461
+#SetOption('warn', 'all')
+
 TICI = os.path.isfile('/TICI')
 AGNOS = TICI
 
 Decider('MD5-timestamp')
 
+SetOption('num_jobs', int(os.cpu_count()/2))
+
 AddOption('--kaitai',
           action='store_true',
           help='Regenerate kaitai struct parsers')

body

@@ -1 +1 @@
-Subproject commit 3aa61382b7ea9328cab7f1a2fe1ec701dffd018f
+Subproject commit 61ace31efad27ae0d6d86888842f82bc92545e72

cereal

@@ -1 +1 @@
-Subproject commit bceb8b942d3e622c2476e197102950efc4fe0bfd
+Subproject commit d81d86e7cd83d1eb40314964a4d194231381d557

common/logging_extra.py

@@ -65,7 +65,7 @@ class SwagFormatter(logging.Formatter):
 
     return record_dict
 
-  def format(self, record): # noqa: A003
+  def format(self, record):
     if self.swaglogger is None:
       raise Exception("must set swaglogger before calling format()")
     return json_robust_dumps(self.format_dict(record))
@@ -95,7 +95,7 @@ class SwagLogFileFormatter(SwagFormatter):
       k += "$a"
     return k, v
 
-  def format(self, record): # noqa: A003
+  def format(self, record):
     if isinstance(record, str):
       v = json.loads(record)
     else:

common/params.cc

@@ -199,7 +199,7 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"UbloxAvailable", PERSISTENT},
     {"UpdateAvailable", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
     {"UpdateFailedCount", CLEAR_ON_MANAGER_START},
-    {"UpdaterAvailableBranches", CLEAR_ON_MANAGER_START},
+    {"UpdaterAvailableBranches", PERSISTENT},
     {"UpdaterCurrentDescription", CLEAR_ON_MANAGER_START},
     {"UpdaterCurrentReleaseNotes", CLEAR_ON_MANAGER_START},
     {"UpdaterFetchAvailable", CLEAR_ON_MANAGER_START},

common/profiler.py

@@ -1,45 +0,0 @@
-import time
-
-class Profiler():
-  def __init__(self, enabled=False):
-    self.enabled = enabled
-    self.cp = {}
-    self.cp_ignored = []
-    self.iter = 0
-    self.start_time = time.time()
-    self.last_time = self.start_time
-    self.tot = 0.
-
-  def reset(self, enabled=False):
-    self.enabled = enabled
-    self.cp = {}
-    self.cp_ignored = []
-    self.iter = 0
-    self.start_time = time.time()
-    self.last_time = self.start_time
-
-  def checkpoint(self, name, ignore=False):
-    # ignore flag needed when benchmarking threads with ratekeeper
-    if not self.enabled:
-      return
-    tt = time.time()
-    if name not in self.cp:
-      self.cp[name] = 0.
-      if ignore:
-        self.cp_ignored.append(name)
-    self.cp[name] += tt - self.last_time
-    if not ignore:
-      self.tot += tt - self.last_time
-    self.last_time = tt
-
-  def display(self):
-    if not self.enabled:
-      return
-    self.iter += 1
-    print("******* Profiling %d *******" % self.iter)
-    for n, ms in sorted(self.cp.items(), key=lambda x: -x[1]):
-      if n in self.cp_ignored:
-        print("%30s: %9.2f  avg: %7.2f  percent: %3.0f   IGNORED" % (n, ms*1000.0, ms*1000.0/self.iter, ms/self.tot*100))
-      else:
-        print("%30s: %9.2f  avg: %7.2f  percent: %3.0f" % (n, ms*1000.0, ms*1000.0/self.iter, ms/self.tot*100))
-    print(f"Iter clock: {self.tot / self.iter:2.6f}   TOTAL: {self.tot:2.2f}")

common/realtime.py

@@ -78,7 +78,7 @@ class Ratekeeper:
       time.sleep(self._remaining)
     return lagged
 
-  # this only monitor the cumulative lag, but does not enforce a rate
+  # Monitors the cumulative lag, but does not enforce a rate
   def monitor_time(self) -> bool:
     prev = self._last_monitor_time
     self._last_monitor_time = time.monotonic()

docs/BOUNTIES.md

@@ -10,11 +10,11 @@ Get paid to improve openpilot!
 * open a ticket at [comma.ai/support](https://comma.ai/support/shop-order) with links to your PRs to claim
 * get an extra 20% if you redeem your bounty in [comma shop](https://comma.ai/shop) credit
 
-New bounties can be proposed in the **#contributing** channel in Discord.
+New bounties can be proposed in the [**#contributing**](https://discord.com/channels/469524606043160576/1183173332531687454) channel in Discord.
 
 ## Issue bounties
 
-We've tagged bounty eligible issues across openpilot and the rest of our repos; check out all the open ones [here](https://github.com/orgs/commaai/projects/26/views/1). These bounties roughly work out like this:
+We've tagged bounty-eligible issues across openpilot and the rest of our repos; check out all the open ones [here](https://github.com/orgs/commaai/projects/26/views/1). These bounties roughly work out like this:
 * **$100** - a few hours of work for an experienced openpilot developer; a good intro for someone new to openpilot
 * **$300** - a day of work for an experienced openpilot developer
 * **$500** - a few days of work for an experienced openpilot developer

docs/CARS.md

@@ -4,7 +4,7 @@
 
 A supported vehicle is one that just works when you install a comma device. All supported cars provide a better experience than any stock system. Supported vehicles reference the US market unless otherwise specified.
 
-# 273 Supported Cars
+# 274 Supported Cars
 
 |Make|Model|Supported Package|ACC|No ACC accel below|No ALC below|Steering Torque|Resume from stop|<a href="##"><img width=2000></a>Hardware Needed<br>&nbsp;|Video|
 |---|---|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|
@@ -132,7 +132,8 @@ A supported vehicle is one that just works when you install a comma device. All
 |Kia|Niro EV 2021|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai C connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Niro EV 2021">Buy Here</a></sub></details>|<a href="https://www.youtube.com/watch?v=lT7zcG6ZpGo" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Kia|Niro EV 2022|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai H connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Niro EV 2022">Buy Here</a></sub></details>|<a href="https://www.youtube.com/watch?v=lT7zcG6ZpGo" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Kia|Niro EV 2023[<sup>6</sup>](#footnotes)|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai A connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Niro EV 2023">Buy Here</a></sub></details>||
-|Kia|Niro Hybrid 2021-22|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai F connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Niro Hybrid 2021-22">Buy Here</a></sub></details>||
+|Kia|Niro Hybrid 2021|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai D connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Niro Hybrid 2021">Buy Here</a></sub></details>||
+|Kia|Niro Hybrid 2022|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai F connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Niro Hybrid 2022">Buy Here</a></sub></details>||
 |Kia|Niro Hybrid 2023[<sup>6</sup>](#footnotes)|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai A connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Niro Hybrid 2023">Buy Here</a></sub></details>||
 |Kia|Niro Plug-in Hybrid 2018-19|All|Stock|10 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai C connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Niro Plug-in Hybrid 2018-19">Buy Here</a></sub></details>||
 |Kia|Niro Plug-in Hybrid 2020|All|Stock|0 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai D connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Niro Plug-in Hybrid 2020">Buy Here</a></sub></details>||
@@ -239,12 +240,12 @@ A supported vehicle is one that just works when you install a comma device. All
 |Toyota|RAV4 2017-18|All|openpilot available[<sup>2</sup>](#footnotes)|19 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Toyota&model=RAV4 2017-18">Buy Here</a></sub></details>||
 |Toyota|RAV4 2019-21|All|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Toyota&model=RAV4 2019-21">Buy Here</a></sub></details>|<a href="https://www.youtube.com/watch?v=wJxjDd42gGA" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Toyota|RAV4 2022|All|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Toyota&model=RAV4 2022">Buy Here</a></sub></details>||
-|Toyota|RAV4 2023|All|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Toyota&model=RAV4 2023">Buy Here</a></sub></details>||
+|Toyota|RAV4 2023-24|All|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Toyota&model=RAV4 2023-24">Buy Here</a></sub></details>||
 |Toyota|RAV4 Hybrid 2016|Toyota Safety Sense P|openpilot available[<sup>2</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Toyota&model=RAV4 Hybrid 2016">Buy Here</a></sub></details>|<a href="https://youtu.be/LhT5VzJVfNI?t=26" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Toyota|RAV4 Hybrid 2017-18|All|openpilot available[<sup>2</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Toyota&model=RAV4 Hybrid 2017-18">Buy Here</a></sub></details>|<a href="https://youtu.be/LhT5VzJVfNI?t=26" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Toyota|RAV4 Hybrid 2019-21|All|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Toyota&model=RAV4 Hybrid 2019-21">Buy Here</a></sub></details>||
 |Toyota|RAV4 Hybrid 2022|All|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Toyota&model=RAV4 Hybrid 2022">Buy Here</a></sub></details>|<a href="https://youtu.be/U0nH9cnrFB0" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
-|Toyota|RAV4 Hybrid 2023|All|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Toyota&model=RAV4 Hybrid 2023">Buy Here</a></sub></details>||
+|Toyota|RAV4 Hybrid 2023-24|All|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Toyota&model=RAV4 Hybrid 2023-24">Buy Here</a></sub></details>||
 |Toyota|Sienna 2018-20|All|openpilot available[<sup>2</sup>](#footnotes)|19 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Toyota&model=Sienna 2018-20">Buy Here</a></sub></details>|<a href="https://www.youtube.com/watch?v=q1UPOo4Sh68" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Volkswagen|Arteon 2018-23|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,13</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 J533 connector<br>- 1 USB-C coupler<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Arteon 2018-23">Buy Here</a></sub></details>|<a href="https://youtu.be/FAomFKPFlDA" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Volkswagen|Arteon eHybrid 2020-23|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,13</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 J533 connector<br>- 1 USB-C coupler<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Arteon eHybrid 2020-23">Buy Here</a></sub></details>|<a href="https://youtu.be/FAomFKPFlDA" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|

docs/CONTRIBUTING.md

@@ -11,7 +11,7 @@ Our software is open source so you can solve your own problems without needing h
 
 ## What contributions are we looking for?
 
-**openpilot's priorities are [safety](SAFETY.md), stability, quality, and features, in that order.** openpilot is part of comma's mission to *solve self-driving cars while delivering shippable intermediaries*, and **all** developoment is towards that goal. 
+**openpilot's priorities are [safety](SAFETY.md), stability, quality, and features, in that order.** openpilot is part of comma's mission to *solve self-driving cars while delivering shippable intermediaries*, and **all** development is towards that goal. 
 
 ### What gets merged?
 

opendbc

@@ -1 +1 @@
-Subproject commit ff235b706b46c01ca34661e91cbbf769fe782ec9
+Subproject commit d47ab8751ffa64fe15ce5c1767e04193b06bd189

panda

@@ -1 +1 @@
-Subproject commit 114b85a649341d55d6beb36d7414eda5e6d324a2
+Subproject commit d66161966d8468223b645c8eba1324e9a49de916

pyproject.toml

@@ -21,6 +21,7 @@ testpaths = [
   "selfdrive/thermald",
   "selfdrive/test/longitudinal_maneuvers",
   "selfdrive/test/process_replay/test_fuzzy.py",
+  "system/camerad",
   "system/hardware/tici",
   "system/loggerd",
   "system/proclogd",
@@ -87,8 +88,8 @@ json-rpc = "*"
 libusb1 = "*"
 numpy = "*"
 onnx = ">=1.14.0"
-onnxruntime = { version = ">=1.15.1", platform = "linux", markers = "platform_machine == 'aarch64'" }
+onnxruntime = { version = ">=1.16.3", platform = "linux", markers = "platform_machine == 'aarch64'" }
-onnxruntime-gpu = { version = ">=1.15.1", platform = "linux", markers = "platform_machine == 'x86_64'" }
+onnxruntime-gpu = { version = ">=1.16.3", platform = "linux", markers = "platform_machine == 'x86_64'" }
 psutil = "*"
 pyaudio = "*"
 pycapnp = "*"
@@ -125,7 +126,7 @@ inputs = "*"
 Jinja2 = "*"
 lru-dict = "*"
 matplotlib = "*"
-metadrive-simulator = { git = "https://github.com/metadriverse/metadrive.git", rev ="72e842cd1d025bf676e4af8797a01e4aa282109f", markers = "platform_machine != 'aarch64'" } # no linux/aarch64 wheels for certain dependencies
+metadrive-simulator = { git = "https://github.com/metadriverse/metadrive.git", rev ="main", markers = "platform_machine != 'aarch64'" } # no linux/aarch64 wheels for certain dependencies
 mpld3 = "*"
 mypy = "*"
 myst-parser = "*"

rednose_repo

@@ -1 +1 @@
-Subproject commit 44e8a891a2810f274a1fa980775155d9463e87b9
+Subproject commit 18b91458fd396530d43e1a2fe9a3ac9055fa9109

release/files_common

@@ -291,11 +291,8 @@ system/camerad/main.cc
 system/camerad/snapshot/*
 system/camerad/cameras/camera_common.h
 system/camerad/cameras/camera_common.cc
-system/camerad/sensors/ar0231.cc
+system/camerad/sensors/*.h
-system/camerad/sensors/ar0231_registers.h
+system/camerad/sensors/*.cc
-system/camerad/sensors/ox03c10.cc
-system/camerad/sensors/ox03c10_registers.h
-system/camerad/sensors/sensor.h
 
 selfdrive/manager/__init__.py
 selfdrive/manager/build.py

selfdrive/athena/athenad.py

@@ -79,7 +79,7 @@ class UploadItem:
   url: str
   headers: Dict[str, str]
   created_at: int
-  id: Optional[str] # noqa: A003 (to match the response from the remote server)
+  id: Optional[str]
   retry_count: int = 0
   current: bool = False
   progress: float = 0
@@ -637,6 +637,8 @@ def ws_proxy_recv(ws: WebSocket, local_sock: socket.socket, ssock: socket.socket
   while not (end_event.is_set() or global_end_event.is_set()):
     try:
       data = ws.recv()
+      if isinstance(data, str):
+        data = data.encode("utf-8")
       local_sock.sendall(data)
     except WebSocketTimeoutException:
       pass
@@ -728,10 +730,11 @@ def ws_manage(ws: WebSocket, end_event: threading.Event) -> None:
     if onroad != onroad_prev:
       onroad_prev = onroad
 
-      sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_USER_TIMEOUT, 16000 if onroad else 0)
+      if sock is not None:
-      sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 7 if onroad else 30)
+        sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_USER_TIMEOUT, 16000 if onroad else 0)
-      sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 7 if onroad else 10)
+        sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 7 if onroad else 30)
-      sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 2 if onroad else 3)
+        sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 7 if onroad else 10)
+        sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 2 if onroad else 3)
 
     if end_event.wait(5):
       break

selfdrive/athena/tests/helpers.py

@@ -1,12 +1,7 @@
 import http.server
-import random
+import threading
-import requests
 import socket
-import time
 from functools import wraps
-from multiprocessing import Process
-
-from openpilot.common.timeout import Timeout
 
 
 class MockResponse:
@@ -49,35 +44,6 @@ class MockApi():
     return "fake-token"
 
 
-class MockParams():
-  default_params = {
-    "DongleId": b"0000000000000000",
-    "GithubSshKeys": b"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC307aE+nuHzTAgaJhzSf5v7ZZQW9gaperjhCmyPyl4PzY7T1mDGenTlVTN7yoVFZ9UfO9oMQqo0n1OwDIiqbIFxqnhrHU0cYfj88rI85m5BEKlNu5RdaVTj1tcbaPpQc5kZEolaI1nDDjzV0lwS7jo5VYDHseiJHlik3HH1SgtdtsuamGR2T80q1SyW+5rHoMOJG73IH2553NnWuikKiuikGHUYBd00K1ilVAK2xSiMWJp55tQfZ0ecr9QjEsJ+J/efL4HqGNXhffxvypCXvbUYAFSddOwXUPo5BTKevpxMtH+2YrkpSjocWA04VnTYFiPG6U4ItKmbLOTFZtPzoez private", # noqa: E501
-    "GithubUsername": b"commaci",
-    "GsmMetered": True,
-    "AthenadUploadQueue": '[]',
-  }
-  params = default_params.copy()
-
-  @staticmethod
-  def restore_defaults():
-    MockParams.params = MockParams.default_params.copy()
-
-  def get_bool(self, k):
-    return bool(MockParams.params.get(k))
-
-  def get(self, k, encoding=None):
-    ret = MockParams.params.get(k)
-    if ret is not None and encoding is not None:
-      ret = ret.decode(encoding)
-    return ret
-
-  def put(self, k, v):
-    if k not in MockParams.params:
-      raise KeyError(f"key: {k} not in MockParams")
-    MockParams.params[k] = v
-
-
 class MockWebsocket():
   def __init__(self, recv_queue, send_queue):
     self.recv_queue = recv_queue
@@ -101,30 +67,23 @@ class HTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
     self.end_headers()
 
 
-def with_http_server(func):
+def with_http_server(func, handler=http.server.BaseHTTPRequestHandler, setup=None):
   @wraps(func)
   def inner(*args, **kwargs):
-    with Timeout(2, 'HTTP Server did not start'):
+    host = '127.0.0.1'
-      p = None
+    server = http.server.HTTPServer((host, 0), handler)
-      host = '127.0.0.1'
+    port = server.server_port
-      while p is None or p.exitcode is not None:
+    t = threading.Thread(target=server.serve_forever)
-        port = random.randrange(40000, 50000)
+    t.start()
-        p = Process(target=http.server.test,
+
-                    kwargs={'port': port, 'HandlerClass': HTTPRequestHandler, 'bind': host})
+    if setup is not None:
-        p.start()
+      setup(host, port)
-        time.sleep(0.1)
-
-    with Timeout(2, 'HTTP Server seeding failed'):
-      while True:
-        try:
-          requests.put(f'http://{host}:{port}/qlog.bz2', data='', timeout=10)
-          break
-        except requests.exceptions.ConnectionError:
-          time.sleep(0.1)
 
     try:
       return func(*args, f'http://{host}:{port}', **kwargs)
     finally:
-      p.terminate()
+      server.shutdown()
+      server.server_close()
+      t.join()
 
   return inner

selfdrive/athena/tests/test_athenad.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+from functools import partial
 import json
 import multiprocessing
 import os
@@ -17,23 +18,50 @@ from unittest import mock
 from websocket import ABNF
 from websocket._exceptions import WebSocketConnectionClosedException
 
+from cereal import messaging
+
+from openpilot.common.params import Params
+from openpilot.common.timeout import Timeout
 from openpilot.selfdrive.athena import athenad
 from openpilot.selfdrive.athena.athenad import MAX_RETRY_COUNT, dispatcher
-from openpilot.selfdrive.athena.tests.helpers import MockWebsocket, MockParams, MockApi, EchoSocket, with_http_server
+from openpilot.selfdrive.athena.tests.helpers import MockWebsocket, MockApi, EchoSocket, with_http_server
-from cereal import messaging
 from openpilot.system.hardware.hw import Paths
+from openpilot.selfdrive.athena.tests.helpers import HTTPRequestHandler
+
+
+def seed_athena_server(host, port):
+  with Timeout(2, 'HTTP Server seeding failed'):
+    while True:
+      try:
+        requests.put(f'http://{host}:{port}/qlog.bz2', data='', timeout=10)
+        break
+      except requests.exceptions.ConnectionError:
+        time.sleep(0.1)
+
+
+with_mock_athena = partial(with_http_server, handler=HTTPRequestHandler, setup=seed_athena_server)
 
 
 class TestAthenadMethods(unittest.TestCase):
   @classmethod
   def setUpClass(cls):
     cls.SOCKET_PORT = 45454
-    athenad.Params = MockParams
     athenad.Api = MockApi
     athenad.LOCAL_PORT_WHITELIST = {cls.SOCKET_PORT}
 
   def setUp(self):
-    MockParams.restore_defaults()
+    self.default_params = {
+      "DongleId": "0000000000000000",
+      "GithubSshKeys": b"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC307aE+nuHzTAgaJhzSf5v7ZZQW9gaperjhCmyPyl4PzY7T1mDGenTlVTN7yoVFZ9UfO9oMQqo0n1OwDIiqbIFxqnhrHU0cYfj88rI85m5BEKlNu5RdaVTj1tcbaPpQc5kZEolaI1nDDjzV0lwS7jo5VYDHseiJHlik3HH1SgtdtsuamGR2T80q1SyW+5rHoMOJG73IH2553NnWuikKiuikGHUYBd00K1ilVAK2xSiMWJp55tQfZ0ecr9QjEsJ+J/efL4HqGNXhffxvypCXvbUYAFSddOwXUPo5BTKevpxMtH+2YrkpSjocWA04VnTYFiPG6U4ItKmbLOTFZtPzoez private", # noqa: E501
+      "GithubUsername": b"commaci",
+      "AthenadUploadQueue": '[]',
+    }
+
+    self.params = Params()
+    for k, v in self.default_params.items():
+      self.params.put(k, v)
+    self.params.put_bool("GsmMetered", True)
+
     athenad.upload_queue = queue.Queue()
     athenad.cur_upload_items.clear()
     athenad.cancelled_uploads.clear()
@@ -138,7 +166,7 @@ class TestAthenadMethods(unittest.TestCase):
       self.assertEqual(athenad.strip_bz2_extension(fn), fn[:-4])
 
   @parameterized.expand([(True,), (False,)])
-  @with_http_server
+  @with_mock_athena
   def test_do_upload(self, compress, host):
     # random bytes to ensure rather large object post-compression
     fn = self._create_file('qlog', data=os.urandom(10000 * 1024))
@@ -152,7 +180,7 @@ class TestAthenadMethods(unittest.TestCase):
     resp = athenad._do_upload(item)
     self.assertEqual(resp.status_code, 201)
 
-  @with_http_server
+  @with_mock_athena
   def test_uploadFileToUrl(self, host):
     fn = self._create_file('qlog.bz2')
 
@@ -163,7 +191,7 @@ class TestAthenadMethods(unittest.TestCase):
     self.assertIsNotNone(resp['items'][0].get('id'))
     self.assertEqual(athenad.upload_queue.qsize(), 1)
 
-  @with_http_server
+  @with_mock_athena
   def test_uploadFileToUrl_duplicate(self, host):
     self._create_file('qlog.bz2')
 
@@ -175,12 +203,12 @@ class TestAthenadMethods(unittest.TestCase):
     resp = dispatcher["uploadFileToUrl"]("qlog.bz2", url2, {})
     self.assertEqual(resp, {'enqueued': 0, 'items': []})
 
-  @with_http_server
+  @with_mock_athena
   def test_uploadFileToUrl_does_not_exist(self, host):
     not_exists_resp = dispatcher["uploadFileToUrl"]("does_not_exist.bz2", "http://localhost:1238", {})
     self.assertEqual(not_exists_resp, {'enqueued': 0, 'items': [], 'failed': ['does_not_exist.bz2']})
 
-  @with_http_server
+  @with_mock_athena
   def test_upload_handler(self, host):
     fn = self._create_file('qlog.bz2')
     item = athenad.UploadItem(path=fn, url=f"{host}/qlog.bz2", headers={}, created_at=int(time.time()*1000), id='', allow_cellular=True)
@@ -199,7 +227,7 @@ class TestAthenadMethods(unittest.TestCase):
     finally:
       end_event.set()
 
-  @with_http_server
+  @with_mock_athena
   @mock.patch('requests.put')
   def test_upload_handler_retry(self, host, mock_put):
     for status, retry in ((500, True), (412, False)):
@@ -377,11 +405,11 @@ class TestAthenadMethods(unittest.TestCase):
 
   def test_getSshAuthorizedKeys(self):
     keys = dispatcher["getSshAuthorizedKeys"]()
-    self.assertEqual(keys, MockParams().params["GithubSshKeys"].decode('utf-8'))
+    self.assertEqual(keys, self.default_params["GithubSshKeys"].decode('utf-8'))
 
   def test_getGithubUsername(self):
     keys = dispatcher["getGithubUsername"]()
-    self.assertEqual(keys, MockParams().params["GithubUsername"].decode('utf-8'))
+    self.assertEqual(keys, self.default_params["GithubUsername"].decode('utf-8'))
 
   def test_getVersion(self):
     resp = dispatcher["getVersion"]()

selfdrive/car/README.md

@@ -1,63 +1,3 @@
-# selfdrive/car
-
-Check out [this blog post](https://blog.comma.ai/how-to-write-a-car-port-for-openpilot/) for a high-level overview of porting a car.
-
-## Useful car porting utilities
-
-Testing car ports in your car is very time-consuming. Check out these utilities to do basic checks on your work before running it in your car.
-
-### [Cabana](/tools/cabana/README.md)
-
-View your car's CAN signals through DBC files, which openpilot uses to parse and create messages that talk to the car.
-
-Example:
-```bash
-> tools/cabana/cabana '1bbe6bf2d62f58a8|2022-07-14--17-11-43'
-```
-
-### [selfdrive/debug/auto_fingerprint.py](/selfdrive/debug/auto_fingerprint.py)
-
-Given a route and platform, automatically inserts FW fingerprints from the platform into the correct place in values.py
-
-Example:
-```bash
-> python selfdrive/debug/auto_fingerprint.py '1bbe6bf2d62f58a8|2022-07-14--17-11-43' 'SUBARU OUTBACK 6TH GEN'
-Attempting to add fw version for:  SUBARU OUTBACK 6TH GEN
-```
-
-### [selfdrive/car/tests/test_car_interfaces.py](/selfdrive/car/tests/test_car_interfaces.py)
-
-Finds common bugs for car interfaces, without even requiring a route.
-
-
-#### Example: Typo in signal name
-```bash
-> pytest selfdrive/car/tests/test_car_interfaces.py -k subaru  # replace with the brand you are working on
-
-=====================================================================
-FAILED selfdrive/car/tests/test_car_interfaces.py::TestCarInterfaces::test_car_interfaces_165_SUBARU_LEGACY_7TH_GEN - KeyError: 'CruiseControlOOPS'
-
-```
-
-### [selfdrive/debug/test_car_model.py](/selfdrive/debug/test_car_model.py)
-
-Given a route, runs most of the car interface to check for common errors like missing signals, blocked panda messages, and safety mismatches.
-
-#### Example: panda safety mismatch for gasPressed
-```bash
-> python selfdrive/debug/test_car_model.py '4822a427b188122a|2023-08-14--16-22-21'
-
-=====================================================================
-FAIL: test_panda_safety_carstate (__main__.CarModelTestCase.test_panda_safety_carstate)
-Assert that panda safety matches openpilot's carState
-----------------------------------------------------------------------
-Traceback (most recent call last):
-  File "/home/batman/xx/openpilot/openpilot/selfdrive/car/tests/test_models.py", line 380, in test_panda_safety_carstate
-    self.assertFalse(len(failed_checks), f"panda safety doesn't agree with openpilot: {failed_checks}")
-AssertionError: 1 is not false : panda safety doesn't agree with openpilot: {'gasPressed': 116}
-```
-
-
 ## Car port structure
 
 ### interface.py

selfdrive/car/chrysler/fingerprints.py

@@ -65,6 +65,7 @@ FW_VERSIONS = {
   CAR.JEEP_GRAND_CHEROKEE_2019: {
     (Ecu.combinationMeter, 0x742, None): [
       b'68402971AD',
+      b'68454144AD',
     ],
     (Ecu.srs, 0x744, None): [
       b'68355363AB',
@@ -77,12 +78,15 @@ FW_VERSIONS = {
     ],
     (Ecu.eps, 0x75a, None): [
       b'68453431AA',
+      b'68453433AA',
     ],
     (Ecu.engine, 0x7e0, None): [
       b'05035674AB ',
+      b'68496223AA ',
     ],
     (Ecu.transmission, 0x7e1, None): [
       b'05035707AA',
+      b'68495807AA',
     ],
   },
   CAR.RAM_1500: {
@@ -95,17 +99,26 @@ FW_VERSIONS = {
       b'68294063AI',
       b'68434846AC',
       b'68434858AC',
+      b'68434859AC',
       b'68434860AC',
+      b'68453483AC',
+      b'68453487AD',
+      b'68453491AC',
+      b'68453499AD',
       b'68453503AC',
       b'68453505AC',
+      b'68453505AD',
       b'68453511AC',
       b'68453513AD',
       b'68453514AD',
       b'68510280AG',
+      b'68510282AH',
       b'68510283AG',
       b'68527346AE',
+      b'68527361AD',
       b'68527375AD',
       b'68527382AE',
+      b'68527387AE',
     ],
     (Ecu.srs, 0x744, None): [
       b'68428609AB',
@@ -155,13 +168,18 @@ FW_VERSIONS = {
       b'68312176AE',
       b'68312176AG',
       b'68440789AC',
+      b'68466110AA',
       b'68466110AB',
       b'68469901AA',
+      b'68469907AA',
+      b'68522583AA',
       b'68522583AB',
+      b'68522584AA',
       b'68522585AB',
       b'68552788AA',
       b'68552789AA',
       b'68552790AA',
+      b'68552791AB',
       b'68585106AB',
       b'68585109AB',
       b'68585112AB',
@@ -170,30 +188,52 @@ FW_VERSIONS = {
       b'05036065AE ',
       b'05036066AE ',
       b'05149591AD ',
+      b'05149591AE ',
       b'05149592AE ',
       b'05149846AA ',
       b'05149848AA ',
+      b'68378695AJ ',
+      b'68378696AJ ',
       b'68378701AI ',
       b'68378748AL ',
       b'68378758AM ',
       b'68448163AJ',
       b'68448165AK',
+      b'68455119AC ',
+      b'68455145AC ',
+      b'68455145AE ',
+      b'68455146AC ',
       b'68500630AD',
       b'68500630AE',
+      b'68502719AC ',
+      b'68502722AC ',
+      b'68502734AF ',
+      b'68502740AF ',
+      b'68502742AC ',
       b'68539650AD',
+      b'68539650AF',
+      b'68586101AA ',
+      b'68586105AB ',
     ],
     (Ecu.transmission, 0x7e1, None): [
+      b'05036069AA',
       b'05149536AC',
       b'68360078AL',
       b'68360080AM',
       b'68360081AM',
+      b'68360085AJ',
       b'68360085AL',
       b'68384328AD',
       b'68384332AD',
+      b'68445531AC',
       b'68445533AB',
+      b'68445537AB',
+      b'68484466AC',
       b'68484467AC',
+      b'68484471AC',
       b'68502994AD',
       b'68520867AE',
+      b'68520867AF',
       b'68540431AB',
     ],
   },

selfdrive/car/docs_definitions.py

@@ -244,10 +244,13 @@ class CarInfo:
   # all the parts needed for the supported car
   car_parts: CarParts = field(default_factory=CarParts)
 
+  def __post_init__(self):
+    self.make, self.model, self.years = split_name(self.name)
+    self.year_list = get_year_list(self.years)
+
   def init(self, CP: car.CarParams, all_footnotes: Dict[Enum, int]):
     self.car_name = CP.carName
     self.car_fingerprint = CP.carFingerprint
-    self.make, self.model, self.years = split_name(self.name)
 
     # longitudinal column
     op_long = "Stock"
@@ -309,7 +312,6 @@ class CarInfo:
       self.row[Column.STEERING_TORQUE] = Star.FULL
 
     self.all_footnotes = all_footnotes
-    self.year_list = get_year_list(self.years)
     self.detail_sentence = self.get_detail_sentence(CP)
 
     return self

selfdrive/car/gm/carcontroller.py

@@ -154,21 +154,6 @@ class CarController:
       if self.frame % 10 == 0:
         can_sends.append(gmcan.create_pscm_status(self.packer_pt, CanBus.CAMERA, CS.pscm_status))
 
-    # Show green icon when LKA torque is applied, and
-    # alarming orange icon when approaching torque limit.
-    # If not sent again, LKA icon disappears in about 5 seconds.
-    # Conveniently, sending camera message periodically also works as a keepalive.
-    lka_active = CS.lkas_status == 1
-    lka_critical = lka_active and abs(actuators.steer) > 0.9
-    lka_icon_status = (lka_active, lka_critical)
-
-    # SW_GMLAN not yet on cam harness, no HUD alerts
-    if self.CP.networkLocation != NetworkLocation.fwdCamera and \
-      (self.frame % self.params.CAMERA_KEEPALIVE_STEP == 0 or lka_icon_status != self.lka_icon_status_last):
-      steer_alert = hud_alert in (VisualAlert.steerRequired, VisualAlert.ldw)
-      can_sends.append(gmcan.create_lka_icon_command(CanBus.SW_GMLAN, lka_active, lka_critical, steer_alert))
-      self.lka_icon_status_last = lka_icon_status
-
     new_actuators = actuators.copy()
     new_actuators.steer = self.apply_steer_last / self.params.STEER_MAX
     new_actuators.steerOutputCan = self.apply_steer_last

selfdrive/car/gm/values.py

@@ -139,7 +139,6 @@ class CanBus:
   OBSTACLE = 1
   CAMERA = 2
   CHASSIS = 2
-  SW_GMLAN = 3
   LOOPBACK = 128
   DROPPED = 192
 

selfdrive/car/honda/fingerprints.py

@@ -218,6 +218,7 @@ FW_VERSIONS = {
     (Ecu.fwdRadar, 0x18dab0f1, None): [
       b'36802-TWA-A070\x00\x00',
       b'36802-TWA-A080\x00\x00',
+      b'36802-TWA-A210\x00\x00',
       b'36802-TWA-A330\x00\x00',
       b'36802-TWB-H060\x00\x00',
     ],
@@ -574,6 +575,7 @@ FW_VERSIONS = {
       b'37805-5PA-6630\x00\x00',
       b'37805-5PA-6640\x00\x00',
       b'37805-5PA-7630\x00\x00',
+      b'37805-5PA-9530\x00\x00',
       b'37805-5PA-9630\x00\x00',
       b'37805-5PA-9640\x00\x00',
       b'37805-5PA-9730\x00\x00',
@@ -634,6 +636,7 @@ FW_VERSIONS = {
       b'46114-TMC-U020\x00\x00',
     ],
     (Ecu.combinationMeter, 0x18da60f1, None): [
+      b'78109-TLA-A020\x00\x00',
       b'78109-TLA-A110\x00\x00',
       b'78109-TLA-A120\x00\x00',
       b'78109-TLA-A210\x00\x00',
@@ -744,9 +747,6 @@ FW_VERSIONS = {
       b'78109-TPG-A110\x00\x00',
       b'78109-TPG-A210\x00\x00',
     ],
-    (Ecu.hud, 0x18da61f1, None): [
-      b'78209-TLA-X010\x00\x00',
-    ],
     (Ecu.fwdRadar, 0x18dab0f1, None): [
       b'36802-TMB-H040\x00\x00',
       b'36802-TPA-E040\x00\x00',
@@ -931,7 +931,9 @@ FW_VERSIONS = {
       b'28101-5EZ-A060\x00\x00',
       b'28101-5EZ-A100\x00\x00',
       b'28101-5EZ-A210\x00\x00',
+      b'28101-5EZ-A330\x00\x00',
       b'28101-5EZ-A430\x00\x00',
+      b'28101-5EZ-A500\x00\x00',
       b'28101-5EZ-A600\x00\x00',
       b'28101-5EZ-A700\x00\x00',
     ],
@@ -943,13 +945,19 @@ FW_VERSIONS = {
       b'37805-RLV-B210\x00\x00',
       b'37805-RLV-B220\x00\x00',
       b'37805-RLV-B420\x00\x00',
+      b'37805-RLV-B430\x00\x00',
+      b'37805-RLV-B720\x00\x00',
       b'37805-RLV-C430\x00\x00',
       b'37805-RLV-C510\x00\x00',
       b'37805-RLV-C520\x00\x00',
       b'37805-RLV-C530\x00\x00',
       b'37805-RLV-C910\x00\x00',
       b'37805-RLV-F120\x00\x00',
+      b'37805-RLV-L090\x00\x00',
       b'37805-RLV-L160\x00\x00',
+      b'37805-RLV-L180\x00\x00',
+      b'37805-RLV-L410\x00\x00',
+      b'37805-RLV-L850\x00\x00',
     ],
     (Ecu.gateway, 0x18daeff1, None): [
       b'38897-TG7-A030\x00\x00',
@@ -983,6 +991,7 @@ FW_VERSIONS = {
       b'36161-TGS-A030\x00\x00',
       b'36161-TGS-A130\x00\x00',
       b'36161-TGS-A220\x00\x00',
+      b'36161-TGS-A320\x00\x00',
       b'36161-TGT-A030\x00\x00',
       b'36161-TGT-A130\x00\x00',
     ],
@@ -1019,7 +1028,9 @@ FW_VERSIONS = {
       b'78109-TG8-AJ10\x00\x00',
       b'78109-TG8-AJ20\x00\x00',
       b'78109-TG8-AK20\x00\x00',
+      b'78109-TGS-AB10\x00\x00',
       b'78109-TGS-AC10\x00\x00',
+      b'78109-TGS-AD10\x00\x00',
       b'78109-TGS-AJ20\x00\x00',
       b'78109-TGS-AK20\x00\x00',
       b'78109-TGS-AP20\x00\x00',
@@ -1072,6 +1083,7 @@ FW_VERSIONS = {
       b'37805-5YF-A420\x00\x00',
       b'37805-5YF-A430\x00\x00',
       b'37805-5YF-A750\x00\x00',
+      b'37805-5YF-A760\x00\x00',
       b'37805-5YF-A850\x00\x00',
       b'37805-5YF-A870\x00\x00',
       b'37805-5YF-AD20\x00\x00',
@@ -1079,6 +1091,7 @@ FW_VERSIONS = {
       b'37805-5YF-C220\x00\x00',
       b'37805-5YF-C410\x00\x00',
       b'37805-5YF-C420\x00\x00',
+      b'37805-5YF-C430\x00\x00',
     ],
     (Ecu.vsa, 0x18da28f1, None): [
       b'57114-TJB-A030\x00\x00',
@@ -1121,6 +1134,7 @@ FW_VERSIONS = {
       b'78109-TJB-AS10\x00\x00',
       b'78109-TJB-AU10\x00\x00',
       b'78109-TJB-AW10\x00\x00',
+      b'78109-TJC-A240\x00\x00',
       b'78109-TJC-A420\x00\x00',
       b'78109-TJC-AA10\x00\x00',
       b'78109-TJC-AD10\x00\x00',

selfdrive/car/hyundai/carstate.py

@@ -147,8 +147,9 @@ class CarState(CarStateBase):
       aeb_src = "FCA11" if self.CP.flags & HyundaiFlags.USE_FCA.value else "SCC12"
       aeb_sig = "FCA_CmdAct" if self.CP.flags & HyundaiFlags.USE_FCA.value else "AEB_CmdAct"
       aeb_warning = cp_cruise.vl[aeb_src]["CF_VSM_Warn"] != 0
+      scc_warning = cp_cruise.vl["SCC12"]["TakeOverReq"] == 1  # sometimes only SCC system shows an FCW
       aeb_braking = cp_cruise.vl[aeb_src]["CF_VSM_DecCmdAct"] != 0 or cp_cruise.vl[aeb_src][aeb_sig] != 0
-      ret.stockFcw = aeb_warning and not aeb_braking
+      ret.stockFcw = (aeb_warning or scc_warning) and not aeb_braking
       ret.stockAeb = aeb_warning and aeb_braking
 
     if self.CP.enableBsm:

selfdrive/car/hyundai/values.py

@@ -251,7 +251,8 @@ CAR_INFO: Dict[str, Optional[Union[HyundaiCarInfo, List[HyundaiCarInfo]]]] = {
   ],
   CAR.KIA_NIRO_PHEV_2022: HyundaiCarInfo("Kia Niro Plug-in Hybrid 2022", "All", car_parts=CarParts.common([CarHarness.hyundai_f])),
   CAR.KIA_NIRO_HEV_2021: [
-    HyundaiCarInfo("Kia Niro Hybrid 2021-22", car_parts=CarParts.common([CarHarness.hyundai_f])),  # TODO: 2021 could be hyundai_d, verify
+    HyundaiCarInfo("Kia Niro Hybrid 2021", car_parts=CarParts.common([CarHarness.hyundai_d])),
+    HyundaiCarInfo("Kia Niro Hybrid 2022", car_parts=CarParts.common([CarHarness.hyundai_f])),
   ],
   CAR.KIA_NIRO_HEV_2ND_GEN: HyundaiCarInfo("Kia Niro Hybrid 2023", car_parts=CarParts.common([CarHarness.hyundai_a])),
   CAR.KIA_OPTIMA_G4: HyundaiCarInfo("Kia Optima 2017", "Advanced Smart Cruise Control",

selfdrive/car/subaru/fingerprints.py

@@ -8,7 +8,6 @@ FW_VERSIONS = {
     (Ecu.abs, 0x7b0, None): [
       b'\xa5 \x19\x02\x00',
       b'\xa5 !\x02\x00',
-      b'\xf1\x82\xa5 \x19\x02\x00',
     ],
     (Ecu.eps, 0x746, None): [
       b'\x05\xc0\xd0\x00',
@@ -25,10 +24,6 @@ FW_VERSIONS = {
     (Ecu.engine, 0x7e0, None): [
       b'\xbb,\xa0t\x07',
       b'\xd1,\xa0q\x07',
-      b'\xf1\x82\xbb,\xa0t\x07',
-      b'\xf1\x82\xbb,\xa0t\x87',
-      b'\xf1\x82\xd1,\xa0q\x07',
-      b'\xf1\x82\xd9,\xa0@\x07',
     ],
     (Ecu.transmission, 0x7e1, None): [
       b'\x00\xfe\xf7\x00\x00',
@@ -59,7 +54,7 @@ FW_VERSIONS = {
       b'\xa1  \x02\x01',
       b'\xa1  \x02\x02',
       b'\xa1  \x03\x03',
-      b'\xa1\\  x04\x01',
+      b'\xa1  \x04\x01',
     ],
     (Ecu.eps, 0x746, None): [
       b'\x9b\xc0\x11\x00',
@@ -96,7 +91,6 @@ FW_VERSIONS = {
       b'\xa2 \x193\x00',
       b'\xa2 \x194\x00',
       b'\xa2 \x19`\x00',
-      b'\xf1\x00\xb2\x06\x04',
     ],
     (Ecu.eps, 0x746, None): [
       b'z\xc0\x00\x00',
@@ -170,7 +164,6 @@ FW_VERSIONS = {
       b'\xa2 !3\x00',
       b'\xa2 !`\x00',
       b'\xa2 !i\x00',
-      b'\xf1\x00\xb2\x06\x04',
     ],
     (Ecu.eps, 0x746, None): [
       b'\n\xc0\x04\x00',
@@ -214,7 +207,6 @@ FW_VERSIONS = {
       b'\xe9\xf5B0\x00',
       b'\xe9\xf6B0\x00',
       b'\xe9\xf6F0\x00',
-      b'\xf1\x00\xd7\x10@',
     ],
   },
   CAR.CROSSTREK_HYBRID: {
@@ -243,7 +235,6 @@ FW_VERSIONS = {
       b'\xa3 \x19&\x00',
       b'\xa3  \x14\x00',
       b'\xa3  \x14\x01',
-      b'\xf1\x00\xbb\r\x05',
     ],
     (Ecu.eps, 0x746, None): [
       b'\x8d\xc0\x00\x00',
@@ -257,7 +248,6 @@ FW_VERSIONS = {
       b'\x00\x00e`\x1f@  ',
       b'\x00\x00e\x97\x00\x00\x00\x00',
       b'\x00\x00e\x97\x1f@ 0',
-      b'\xf1\x00\xac\x02\x00',
     ],
     (Ecu.engine, 0x7e0, None): [
       b'\xb6"`A\x07',
@@ -266,7 +256,6 @@ FW_VERSIONS = {
       b'\xcb"`p\x07',
       b'\xcf"`0\x07',
       b'\xcf"`p\x07',
-      b'\xf1\x00\xa2\x10\n',
     ],
     (Ecu.transmission, 0x7e1, None): [
       b'\x1a\xe6B1\x00',
@@ -299,7 +288,6 @@ FW_VERSIONS = {
     (Ecu.abs, 0x7b0, None): [
       b'm\x97\x14@',
       b'}\x97\x14@',
-      b'\xf1\x00\xbb\x0c\x04',
     ],
     (Ecu.eps, 0x746, None): [
       b'm\xc0\x10\x00',
@@ -316,7 +304,6 @@ FW_VERSIONS = {
       b'\xa7)\xa0q\x07',
       b'\xba"@@\x07',
       b'\xba"@p\x07',
-      b'\xf1\x82\xa7)\xa0q\x07',
     ],
     (Ecu.transmission, 0x7e1, None): [
       b'\x1a\xf6F`\x00',
@@ -386,8 +373,6 @@ FW_VERSIONS = {
       b'\x00\x00c\xb7\x1f@\x10\x16',
       b'\x00\x00c\xd1\x1f@\x10\x17',
       b'\x00\x00c\xec\x1f@ \x04',
-      b'\x00\x00c\xec7@\x04',
-      b'\xf1\x00\xf0\xe0\x0e',
     ],
     (Ecu.engine, 0x7e0, None): [
       b'\xa0"@\x80\x07',
@@ -461,9 +446,9 @@ FW_VERSIONS = {
   },
   CAR.OUTBACK: {
     (Ecu.abs, 0x7b0, None): [
-      b'\xa1 \x06\x02',
       b'\xa1  \x06\x00',
       b'\xa1  \x06\x01',
+      b'\xa1  \x06\x02',
       b'\xa1  \x07\x00',
       b'\xa1  \x07\x02',
       b'\xa1  \x08\x00',
@@ -494,9 +479,6 @@ FW_VERSIONS = {
       b'\xe2"`0\x07',
       b'\xe2"`p\x07',
       b'\xe3,\xa0@\x07',
-      b'\xf1\x82\xbc,\xa0q\x07',
-      b'\xf1\x82\xe2,\xa0@\x07',
-      b'\xf1\x82\xe3,\xa0@\x07',
     ],
     (Ecu.transmission, 0x7e1, None): [
       b'\xa5\xf6D@\x00',
@@ -506,7 +488,6 @@ FW_VERSIONS = {
       b'\xa7\x8e\xf40\x00',
       b'\xa7\xf6D@\x00',
       b'\xa7\xfe\xf4@\x00',
-      b'\xf1\x82\xa7\xf6D@\x00',
     ],
   },
   CAR.FORESTER_2022: {

selfdrive/car/subaru/interface.py

@@ -122,7 +122,7 @@ class CarInterface(CarInterfaceBase):
     ret.openpilotLongitudinalControl = experimental_long and ret.experimentalLongitudinalAvailable
 
     if candidate in GLOBAL_GEN2 and ret.openpilotLongitudinalControl:
-      ret.flags |= SubaruFlags.DISABLE_EYESIGHT
+      ret.flags |= SubaruFlags.DISABLE_EYESIGHT.value
 
     if ret.openpilotLongitudinalControl:
       ret.longitudinalTuning.kpBP = [0., 5., 35.]

selfdrive/car/subaru/subarucan.py

@@ -48,8 +48,7 @@ def create_es_distance(packer, frame, es_distance_msg, bus, pcm_cancel_cmd, long
     values["Cruise_Soft_Disable"] = 0
     values["Cruise_Fault"] = 0
 
-    if brake_cmd:
+    values["Cruise_Brake_Active"] = brake_cmd
-      values["Cruise_Brake_Active"] = 1
 
   if pcm_cancel_cmd:
     values["Cruise_Cancel"] = 1
@@ -153,14 +152,14 @@ def create_es_dashstatus(packer, frame, dashstatus_msg, enabled, long_enabled, l
 
   values["COUNTER"] = frame % 0x10
 
-  if enabled and long_active:
+  if long_enabled:
     values["Cruise_State"] = 0
-    values["Cruise_Activated"] = 1
+    values["Cruise_Activated"] = enabled
     values["Cruise_Disengaged"] = 0
     values["Car_Follow"] = int(lead_visible)
 
-  if long_enabled:
     values["PCB_Off"] = 1 # AEB is not presevered, so show the PCB_Off on dash
+    values["LDW_Off"] = 0
     values["Cruise_Fault"] = 0
 
   # Filter stock LKAS disabled and Keep hands on steering wheel OFF alerts
@@ -186,15 +185,12 @@ def create_es_brake(packer, frame, es_brake_msg, long_enabled, long_active, brak
 
   if long_enabled:
     values["Cruise_Brake_Fault"] = 0
+    values["Cruise_Activated"] = long_active
 
-    if long_active:
+    values["Brake_Pressure"] = brake_value
-      values["Cruise_Activated"] = 1
 
-  values["Brake_Pressure"] = brake_value
+    values["Cruise_Brake_Active"] = brake_value > 0
-
+    values["Cruise_Brake_Lights"] = brake_value >= 70
-  if brake_value > 0:
-    values["Cruise_Brake_Active"] = 1
-    values["Cruise_Brake_Lights"] = 1 if brake_value >= 70 else 0
 
   return packer.make_can_msg("ES_Brake", CanBus.main, values)
 
@@ -204,7 +200,6 @@ def create_es_status(packer, frame, es_status_msg, long_enabled, long_active, cr
     "Signal1",
     "Cruise_Fault",
     "Cruise_RPM",
-    "Signal2",
     "Cruise_Activated",
     "Brake_Lights",
     "Cruise_Hold",
@@ -217,8 +212,7 @@ def create_es_status(packer, frame, es_status_msg, long_enabled, long_active, cr
     values["Cruise_RPM"] = cruise_rpm
     values["Cruise_Fault"] = 0
 
-    if long_active:
+    values["Cruise_Activated"] = long_active
-      values["Cruise_Activated"] = 1
 
   return packer.make_can_msg("ES_Status", CanBus.main, values)
 

selfdrive/car/subaru/tests/test_subaru.py

@@ -0,0 +1,20 @@
+from cereal import car
+import unittest
+from openpilot.selfdrive.car.subaru.fingerprints import FW_VERSIONS
+
+Ecu = car.CarParams.Ecu
+
+ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()}
+
+
+class TestSubaruFingerprint(unittest.TestCase):
+  def test_fw_version_format(self):
+    for platform, fws_per_ecu in FW_VERSIONS.items():
+      for (ecu, _, _), fws in fws_per_ecu.items():
+        fw_size = len(fws[0])
+        for fw in fws:
+          self.assertEqual(len(fw), fw_size, f"{platform} {ecu}: {len(fw)} {fw_size}")
+
+
+if __name__ == "__main__":
+  unittest.main()

selfdrive/car/tests/test_lateral_limits.py

@@ -95,8 +95,8 @@ if __name__ == "__main__":
                 _jerks["down_jerk"] > MAX_LAT_JERK_DOWN
     violation_str = " - VIOLATION" if violation else ""
 
-    print(f"{car_model:{max_car_model_len}} - up jerk: {round(_jerks['up_jerk'], 2):5} \
+    print(f"{car_model:{max_car_model_len}} - up jerk: {round(_jerks['up_jerk'], 2):5} " +
-              m/s^3, down jerk: {round(_jerks['down_jerk'], 2):5} m/s^3{violation_str}")
+          f"m/s^3, down jerk: {round(_jerks['down_jerk'], 2):5} m/s^3{violation_str}")
 
   # exit with test result
   sys.exit(not result.result.wasSuccessful())

selfdrive/car/tests/test_models.py

@@ -21,7 +21,8 @@ from openpilot.selfdrive.car.car_helpers import FRAME_FINGERPRINT, interfaces
 from openpilot.selfdrive.car.honda.values import CAR as HONDA, HONDA_BOSCH
 from openpilot.selfdrive.car.tests.routes import non_tested_cars, routes, CarTestRoute
 from openpilot.selfdrive.controls.controlsd import Controls
-from openpilot.selfdrive.test.openpilotci import get_url
+from openpilot.selfdrive.test.helpers import read_segment_list, sanitize
+from openpilot.tools.lib.openpilotci import get_url
 from openpilot.tools.lib.logreader import LogReader
 from openpilot.tools.lib.route import Route, SegmentName, RouteName
 
@@ -36,9 +37,9 @@ JOB_ID = int(os.environ.get("JOB_ID", "0"))
 INTERNAL_SEG_LIST = os.environ.get("INTERNAL_SEG_LIST", "")
 INTERNAL_SEG_CNT = int(os.environ.get("INTERNAL_SEG_CNT", "0"))
 MAX_EXAMPLES = int(os.environ.get("MAX_EXAMPLES", "50"))
-
 CI = os.environ.get("CI", None) is not None
 
+
 def get_test_cases() -> List[Tuple[str, Optional[CarTestRoute]]]:
   # build list of test cases
   test_cases = []
@@ -52,12 +53,9 @@ def get_test_cases() -> List[Tuple[str, Optional[CarTestRoute]]]:
         test_cases.extend(sorted((c.value, r) for r in routes_by_car.get(c, (None,))))
 
   else:
-    with open(os.path.join(BASEDIR, INTERNAL_SEG_LIST), "r") as f:
+    segment_list = read_segment_list(os.path.join(BASEDIR, INTERNAL_SEG_LIST))
-      seg_list = f.read().splitlines()
+    segment_list = random.sample(segment_list, INTERNAL_SEG_CNT or len(segment_list))
-
+    for platform, segment in segment_list:
-    seg_list_grouped = [(platform[2:], segment) for platform, segment in zip(seg_list[::2], seg_list[1::2], strict=True)]
-    seg_list_grouped = random.sample(seg_list_grouped, INTERNAL_SEG_CNT or len(seg_list_grouped))
-    for platform, segment in seg_list_grouped:
       segment_name = SegmentName(segment)
       test_cases.append((platform, CarTestRoute(segment_name.route_name.canonical_name, platform,
                                                 segment=segment_name.segment_num)))
@@ -68,7 +66,7 @@ def get_test_cases() -> List[Tuple[str, Optional[CarTestRoute]]]:
 class TestCarModelBase(unittest.TestCase):
   car_model: Optional[str] = None
   test_route: Optional[CarTestRoute] = None
-  test_route_on_bucket: bool = True # whether the route is on the preserved CI bucket
+  test_route_on_bucket: bool = True  # whether the route is on the preserved CI bucket
 
   can_msgs: List[capnp.lib.capnp._DynamicStructReader]
   fingerprint: dict[int, dict[int, int]]
@@ -81,19 +79,81 @@ class TestCarModelBase(unittest.TestCase):
       route_name = RouteName(cls.test_route.route)
       return LogReader(f"cd:/{route_name.dongle_id}/{route_name.time_str}/{seg}/rlog.bz2")
     else:
-      # Attempt to use CI bucket first
+      return LogReader(get_url(cls.test_route.route, seg))
-      try:
+
-        return LogReader(get_url(cls.test_route.route, seg))
+  @classmethod
-      except Exception:
+  def get_testing_data_from_logreader(cls, lr):
-        cls.test_route_on_bucket = False
+    lr = sanitize(lr)
+    car_fw = []
+    can_msgs = []
+    cls.elm_frame = None
+    cls.car_safety_mode_frame = None
+    cls.fingerprint = gen_empty_fingerprint()
+    experimental_long = False
+    for msg in lr:
+      if msg.which() == "can":
+        can_msgs.append(msg)
+        if len(can_msgs) <= FRAME_FINGERPRINT:
+          for m in msg.can:
+            if m.src < 64:
+              cls.fingerprint[m.src][m.address] = len(m.dat)
+
+      elif msg.which() == "carParams":
+        car_fw = msg.carParams.carFw
+        if msg.carParams.openpilotLongitudinalControl:
+          experimental_long = True
+        if cls.car_model is None and not cls.ci:
+          cls.car_model = msg.carParams.carFingerprint
+
+      # Log which can frame the panda safety mode left ELM327, for CAN validity checks
+      elif msg.which() == 'pandaStates':
+        for ps in msg.pandaStates:
+          if cls.elm_frame is None and ps.safetyModel != SafetyModel.elm327:
+            cls.elm_frame = len(can_msgs)
+          if cls.car_safety_mode_frame is None and ps.safetyModel not in \
+            (SafetyModel.elm327, SafetyModel.noOutput):
+            cls.car_safety_mode_frame = len(can_msgs)
 
-      # Fallback to public route, which will fail the test_route_on_ci_bucket when running in CI
+      elif msg.which() == 'pandaStateDEPRECATED':
+        if cls.elm_frame is None and msg.pandaStateDEPRECATED.safetyModel != SafetyModel.elm327:
+          cls.elm_frame = len(can_msgs)
+        if cls.car_safety_mode_frame is None and msg.pandaStateDEPRECATED.safetyModel not in \
+          (SafetyModel.elm327, SafetyModel.noOutput):
+          cls.car_safety_mode_frame = len(can_msgs)
+
+    if len(can_msgs) > int(50 / DT_CTRL):
+      return car_fw, can_msgs, experimental_long
+
+    raise Exception("no can data found")
+
+  @classmethod
+  def get_testing_data(cls):
+    test_segs = (2, 1, 0)
+    if cls.test_route.segment is not None:
+      test_segs = (cls.test_route.segment,)
+
+    # Try the primary method first (CI or internal)
+    for seg in test_segs:
       try:
-        return LogReader(Route(cls.test_route.route).log_paths()[seg])
+        lr = cls.get_logreader(seg)
+        return cls.get_testing_data_from_logreader(lr)
       except Exception:
         pass
 
-      raise Exception("Unable to get route. Check that the route is valid, and either public or uploaded to the CI bucket.")
+    # Route is not in CI bucket, assume either user has access (private), or it is public
+    # test_route_on_ci_bucket will fail when running in CI
+    if not len(INTERNAL_SEG_LIST):
+      cls.test_route_on_bucket = False
+
+      for seg in test_segs:
+        try:
+          lr = LogReader(Route(cls.test_route.route).log_paths()[seg])
+          return cls.get_testing_data_from_logreader(lr)
+        except Exception:
+          pass
+
+    raise Exception(f"Route: {repr(cls.test_route.route)} with segments: {test_segs} not found or no CAN msgs found. Is it uploaded and public?")
+
 
   @classmethod
   def setUpClass(cls):
@@ -110,57 +170,7 @@ class TestCarModelBase(unittest.TestCase):
         raise unittest.SkipTest
       raise Exception(f"missing test route for {cls.car_model}")
 
-    test_segs = (2, 1, 0)
+    car_fw, can_msgs, experimental_long = cls.get_testing_data()
-    if cls.test_route.segment is not None:
-      test_segs = (cls.test_route.segment,)
-
-    for seg in test_segs:
-      try:
-        lr = cls.get_logreader(seg)
-      except Exception:
-        continue
-
-      car_fw = []
-      can_msgs = []
-      cls.elm_frame = None
-      cls.car_safety_mode_frame = None
-      cls.fingerprint = gen_empty_fingerprint()
-      experimental_long = False
-      for msg in lr:
-        if msg.which() == "can":
-          can_msgs.append(msg)
-          if len(can_msgs) <= FRAME_FINGERPRINT:
-            for m in msg.can:
-              if m.src < 64:
-                cls.fingerprint[m.src][m.address] = len(m.dat)
-
-        elif msg.which() == "carParams":
-          car_fw = msg.carParams.carFw
-          if msg.carParams.openpilotLongitudinalControl:
-            experimental_long = True
-          if cls.car_model is None and not cls.ci:
-            cls.car_model = msg.carParams.carFingerprint
-
-        # Log which can frame the panda safety mode left ELM327, for CAN validity checks
-        elif msg.which() == 'pandaStates':
-          for ps in msg.pandaStates:
-            if cls.elm_frame is None and ps.safetyModel != SafetyModel.elm327:
-              cls.elm_frame = len(can_msgs)
-            if cls.car_safety_mode_frame is None and ps.safetyModel not in \
-              (SafetyModel.elm327, SafetyModel.noOutput):
-              cls.car_safety_mode_frame = len(can_msgs)
-
-        elif msg.which() == 'pandaStateDEPRECATED':
-          if cls.elm_frame is None and msg.pandaStateDEPRECATED.safetyModel != SafetyModel.elm327:
-            cls.elm_frame = len(can_msgs)
-          if cls.car_safety_mode_frame is None and msg.pandaStateDEPRECATED.safetyModel not in \
-            (SafetyModel.elm327, SafetyModel.noOutput):
-            cls.car_safety_mode_frame = len(can_msgs)
-
-      if len(can_msgs) > int(50 / DT_CTRL):
-        break
-    else:
-      raise Exception(f"Route: {repr(cls.test_route.route)} with segments: {test_segs} not found or no CAN msgs found. Is it uploaded and public?")
 
     # if relay is expected to be open in the route
     cls.openpilot_enabled = cls.car_safety_mode_frame is not None
@@ -466,10 +476,10 @@ class TestCarModelBase(unittest.TestCase):
     failed_checks = {k: v for k, v in checks.items() if v > 0}
     self.assertFalse(len(failed_checks), f"panda safety doesn't agree with openpilot: {failed_checks}")
 
-  @pytest.mark.skipif(not CI, reason="When running in CI we want to make sure all the routes are uploaded to the preserved CI bucket.")
+  @unittest.skipIf(not CI, "Accessing non CI-bucket routes is allowed only when not in CI")
   def test_route_on_ci_bucket(self):
-    assert self.test_route_on_bucket, "Route not on CI bucket. \
+    self.assertTrue(self.test_route_on_bucket, "Route not on CI bucket. " +
-                                       This is fine to fail for WIP car ports, just let us know and we can upload your routes to the CI bucket."
+                    "This is fine to fail for WIP car ports, just let us know and we can upload your routes to the CI bucket.")
 
 
 @parameterized_class(('car_model', 'test_route'), get_test_cases())

selfdrive/car/toyota/fingerprints.py

@@ -23,6 +23,7 @@ FW_VERSIONS = {
     (Ecu.fwdRadar, 0x750, 0xf): [
       b'8821F4702000\x00\x00\x00\x00',
       b'8821F4702100\x00\x00\x00\x00',
+      b'8821F4702300\x00\x00\x00\x00',
     ],
     (Ecu.fwdCamera, 0x750, 0x6d): [
       b'8646F0701100\x00\x00\x00\x00',
@@ -51,6 +52,7 @@ FW_VERSIONS = {
       b'8965B41090\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.engine, 0x700, None): [
+      b'\x01896630725100\x00\x00\x00\x00',
       b'\x01896630725200\x00\x00\x00\x00',
       b'\x01896630725300\x00\x00\x00\x00',
       b'\x01896630735100\x00\x00\x00\x00',
@@ -142,6 +144,8 @@ FW_VERSIONS = {
     (Ecu.dsu, 0x791, None): [
       b'8821F0601200    ',
       b'8821F0601300    ',
+      b'8821F0601400    ',
+      b'8821F0601500    ',
       b'8821F0602000    ',
       b'8821F0603300    ',
       b'8821F0603400    ',
@@ -185,6 +189,8 @@ FW_VERSIONS = {
     (Ecu.fwdRadar, 0x750, 0xf): [
       b'8821F0601200    ',
       b'8821F0601300    ',
+      b'8821F0601400    ',
+      b'8821F0601500    ',
       b'8821F0602000    ',
       b'8821F0603300    ',
       b'8821F0603400    ',
@@ -497,6 +503,7 @@ FW_VERSIONS = {
       b'\x018965B12510\x00\x00\x00\x00\x00\x00',
       b'\x018965B12520\x00\x00\x00\x00\x00\x00',
       b'\x018965B12530\x00\x00\x00\x00\x00\x00',
+      b'\x018965B1254000\x00\x00\x00\x00',
       b'\x018965B1255000\x00\x00\x00\x00',
       b'\x018965B1256000\x00\x00\x00\x00',
       b'8965B12361\x00\x00\x00\x00\x00\x00',
@@ -571,6 +578,7 @@ FW_VERSIONS = {
       b'\x028646F1202000\x00\x00\x00\x008646G2601200\x00\x00\x00\x00',
       b'\x028646F1202100\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
       b'\x028646F1202200\x00\x00\x00\x008646G2601500\x00\x00\x00\x00',
+      b'\x028646F1206000\x00\x00\x00\x008646G2601500\x00\x00\x00\x00',
       b'\x028646F1601100\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
       b'\x028646F1601200\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
       b'\x028646F1601300\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
@@ -578,6 +586,7 @@ FW_VERSIONS = {
       b'\x028646F76020C0\x00\x00\x00\x008646G26011A0\x00\x00\x00\x00',
       b'\x028646F7603100\x00\x00\x00\x008646G2601200\x00\x00\x00\x00',
       b'\x028646F7603200\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
+      b'\x028646F7603300\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
       b'\x028646F7605100\x00\x00\x00\x008646G3304000\x00\x00\x00\x00',
     ],
   },
@@ -588,6 +597,7 @@ FW_VERSIONS = {
       b'\x01896630E43100\x00\x00\x00\x00',
       b'\x01896630E43200\x00\x00\x00\x00',
       b'\x01896630E44200\x00\x00\x00\x00',
+      b'\x01896630E44400\x00\x00\x00\x00',
       b'\x01896630E45000\x00\x00\x00\x00',
       b'\x01896630E45100\x00\x00\x00\x00',
       b'\x01896630E45200\x00\x00\x00\x00',
@@ -728,6 +738,7 @@ FW_VERSIONS = {
       b'\x018966353Q4000\x00\x00\x00\x00',
       b'\x018966353R1100\x00\x00\x00\x00',
       b'\x018966353R7100\x00\x00\x00\x00',
+      b'\x018966353R8000\x00\x00\x00\x00',
       b'\x018966353R8100\x00\x00\x00\x00',
     ],
     (Ecu.engine, 0x7e0, None): [
@@ -792,6 +803,7 @@ FW_VERSIONS = {
       b'\x02896634761100\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
       b'\x02896634761200\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
       b'\x02896634762000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x02896634762100\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
       b'\x02896634763000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
       b'\x02896634763100\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
       b'\x02896634765000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
@@ -800,6 +812,7 @@ FW_VERSIONS = {
       b'\x02896634769100\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
       b'\x02896634769200\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
       b'\x02896634770000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x02896634770100\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
       b'\x02896634774000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
       b'\x02896634774100\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
       b'\x02896634774200\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
@@ -812,6 +825,7 @@ FW_VERSIONS = {
       b'\x03896634759100\x00\x00\x00\x008966A4703000\x00\x00\x00\x00897CF4701003\x00\x00\x00\x00',
       b'\x03896634759200\x00\x00\x00\x008966A4703000\x00\x00\x00\x00897CF4701003\x00\x00\x00\x00',
       b'\x03896634759200\x00\x00\x00\x008966A4703000\x00\x00\x00\x00897CF4701004\x00\x00\x00\x00',
+      b'\x03896634759300\x00\x00\x00\x008966A4703000\x00\x00\x00\x00897CF4701003\x00\x00\x00\x00',
       b'\x03896634759300\x00\x00\x00\x008966A4703000\x00\x00\x00\x00897CF4701004\x00\x00\x00\x00',
       b'\x03896634760000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00897CF4701002\x00\x00\x00\x00',
       b'\x03896634760000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00897CF4701003\x00\x00\x00\x00',
@@ -1123,9 +1137,11 @@ FW_VERSIONS = {
   CAR.RAV4_TSS2_2023: {
     (Ecu.abs, 0x7b0, None): [
       b'\x01F15260R450\x00\x00\x00\x00\x00\x00',
+      b'\x01F15260R51000\x00\x00\x00\x00',
       b'\x01F15264283200\x00\x00\x00\x00',
       b'\x01F15264283300\x00\x00\x00\x00',
       b'\x01F152642F1000\x00\x00\x00\x00',
+      b'\x01F152642F8000\x00\x00\x00\x00',
     ],
     (Ecu.eps, 0x7a1, None): [
       b'\x028965B0R11000\x00\x00\x00\x008965B0R12000\x00\x00\x00\x00',
@@ -1137,6 +1153,7 @@ FW_VERSIONS = {
       b'\x01896634AE1001\x00\x00\x00\x00',
       b'\x01896634AF0000\x00\x00\x00\x00',
       b'\x01896634AJ2000\x00\x00\x00\x00',
+      b'\x01896634AL5000\x00\x00\x00\x00',
     ],
     (Ecu.fwdRadar, 0x750, 0xf): [
       b'\x018821F0R03100\x00\x00\x00\x00',
@@ -1144,6 +1161,7 @@ FW_VERSIONS = {
     (Ecu.fwdCamera, 0x750, 0x6d): [
       b'\x028646F0R05100\x00\x00\x00\x008646G0R02100\x00\x00\x00\x00',
       b'\x028646F0R05200\x00\x00\x00\x008646G0R02200\x00\x00\x00\x00',
+      b'\x028646F0R11000\x00\x00\x00\x008646G0R04000\x00\x00\x00\x00',
     ],
   },
   CAR.SIENNA: {
@@ -1237,6 +1255,7 @@ FW_VERSIONS = {
       b'\x018821F3301200\x00\x00\x00\x00',
       b'\x018821F3301300\x00\x00\x00\x00',
       b'\x018821F3301400\x00\x00\x00\x00',
+      b'\x018821F6201200\x00\x00\x00\x00',
       b'\x018821F6201300\x00\x00\x00\x00',
       b'\x018821F6201400\x00\x00\x00\x00',
     ],
@@ -1249,6 +1268,7 @@ FW_VERSIONS = {
       b'\x028646F3304200\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
       b'\x028646F3304300\x00\x00\x00\x008646G2601500\x00\x00\x00\x00',
       b'\x028646F3309100\x00\x00\x00\x008646G3304000\x00\x00\x00\x00',
+      b'\x028646F3309100\x00\x00\x00\x008646G5301200\x00\x00\x00\x00',
     ],
   },
   CAR.LEXUS_ES: {
@@ -1319,6 +1339,7 @@ FW_VERSIONS = {
       b'\x01896637851000\x00\x00\x00\x00',
       b'\x01896637852000\x00\x00\x00\x00',
       b'\x01896637854000\x00\x00\x00\x00',
+      b'\x01896637873000\x00\x00\x00\x00',
       b'\x01896637878000\x00\x00\x00\x00',
     ],
     (Ecu.engine, 0x7e0, None): [
@@ -1391,6 +1412,7 @@ FW_VERSIONS = {
   CAR.LEXUS_RC: {
     (Ecu.engine, 0x700, None): [
       b'\x01896632461100\x00\x00\x00\x00',
+      b'\x01896632478100\x00\x00\x00\x00',
       b'\x01896632478200\x00\x00\x00\x00',
     ],
     (Ecu.engine, 0x7e0, None): [
@@ -1439,6 +1461,7 @@ FW_VERSIONS = {
       b'\x018966348R1300\x00\x00\x00\x00',
       b'\x018966348R8500\x00\x00\x00\x00',
       b'\x018966348W1300\x00\x00\x00\x00',
+      b'\x018966348W2300\x00\x00\x00\x00',
     ],
     (Ecu.abs, 0x7b0, None): [
       b'F152648472\x00\x00\x00\x00\x00\x00',
@@ -1483,6 +1506,7 @@ FW_VERSIONS = {
       b'\x02348Q4000\x00\x00\x00\x00\x00\x00\x00\x00A4802000\x00\x00\x00\x00\x00\x00\x00\x00',
       b'\x02348Q4100\x00\x00\x00\x00\x00\x00\x00\x00A4802000\x00\x00\x00\x00\x00\x00\x00\x00',
       b'\x02348T1100\x00\x00\x00\x00\x00\x00\x00\x00A4802000\x00\x00\x00\x00\x00\x00\x00\x00',
+      b'\x02348T1200\x00\x00\x00\x00\x00\x00\x00\x00A4802000\x00\x00\x00\x00\x00\x00\x00\x00',
       b'\x02348T3000\x00\x00\x00\x00\x00\x00\x00\x00A4802000\x00\x00\x00\x00\x00\x00\x00\x00',
       b'\x02348V6000\x00\x00\x00\x00\x00\x00\x00\x00A4802000\x00\x00\x00\x00\x00\x00\x00\x00',
       b'\x02348Z3000\x00\x00\x00\x00\x00\x00\x00\x00A4802000\x00\x00\x00\x00\x00\x00\x00\x00',
@@ -1503,6 +1527,7 @@ FW_VERSIONS = {
     (Ecu.eps, 0x7a1, None): [
       b'8965B0E011\x00\x00\x00\x00\x00\x00',
       b'8965B0E012\x00\x00\x00\x00\x00\x00',
+      b'8965B48102\x00\x00\x00\x00\x00\x00',
       b'8965B48111\x00\x00\x00\x00\x00\x00',
       b'8965B48112\x00\x00\x00\x00\x00\x00',
     ],
@@ -1638,6 +1663,7 @@ FW_VERSIONS = {
       b'8965B58052\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.abs, 0x7b0, None): [
+      b'F152658320\x00\x00\x00\x00\x00\x00',
       b'F152658341\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.fwdRadar, 0x750, 0xf): [

selfdrive/car/toyota/values.py

@@ -180,8 +180,8 @@ CAR_INFO: Dict[str, Union[ToyotaCarInfo, List[ToyotaCarInfo]]] = {
     ToyotaCarInfo("Toyota RAV4 Hybrid 2022", video_link="https://youtu.be/U0nH9cnrFB0"),
   ],
   CAR.RAV4_TSS2_2023: [
-    ToyotaCarInfo("Toyota RAV4 2023"),
+    ToyotaCarInfo("Toyota RAV4 2023-24"),
-    ToyotaCarInfo("Toyota RAV4 Hybrid 2023"),
+    ToyotaCarInfo("Toyota RAV4 Hybrid 2023-24"),
   ],
   CAR.MIRAI: ToyotaCarInfo("Toyota Mirai 2021"),
   CAR.SIENNA: ToyotaCarInfo("Toyota Sienna 2018-20", video_link="https://www.youtube.com/watch?v=q1UPOo4Sh68", min_enable_speed=MIN_ACC_SPEED),

selfdrive/car/volkswagen/carcontroller.py

@@ -5,7 +5,7 @@ from openpilot.common.conversions import Conversions as CV
 from openpilot.common.realtime import DT_CTRL
 from openpilot.selfdrive.car import apply_driver_steer_torque_limits
 from openpilot.selfdrive.car.volkswagen import mqbcan, pqcan
-from openpilot.selfdrive.car.volkswagen.values import CANBUS, PQ_CARS, CarControllerParams
+from openpilot.selfdrive.car.volkswagen.values import CANBUS, PQ_CARS, CarControllerParams, VolkswagenFlags
 
 VisualAlert = car.CarControl.HUDControl.VisualAlert
 LongCtrlState = car.CarControl.Actuators.LongControlState
@@ -65,13 +65,22 @@ class CarController:
       self.apply_steer_last = apply_steer
       can_sends.append(self.CCS.create_steering_control(self.packer_pt, CANBUS.pt, apply_steer, hca_enabled))
 
+      if self.CP.flags & VolkswagenFlags.STOCK_HCA_PRESENT:
+        # Pacify VW Emergency Assist driver inactivity detection by changing its view of driver steering input torque
+        # to the greatest of actual driver input or 2x openpilot's output (1x openpilot output is not enough to
+        # consistently reset inactivity detection on straight level roads). See commaai/openpilot#23274 for background.
+        ea_simulated_torque = clip(apply_steer * 2, -self.CCP.STEER_MAX, self.CCP.STEER_MAX)
+        if abs(CS.out.steeringTorque) > abs(ea_simulated_torque):
+          ea_simulated_torque = CS.out.steeringTorque
+        can_sends.append(self.CCS.create_eps_update(self.packer_pt, CANBUS.cam, CS.eps_stock_values, ea_simulated_torque))
+
     # **** Acceleration Controls ******************************************** #
 
     if self.frame % self.CCP.ACC_CONTROL_STEP == 0 and self.CP.openpilotLongitudinalControl:
       acc_control = self.CCS.acc_control_value(CS.out.cruiseState.available, CS.out.accFaulted, CC.longActive)
       accel = clip(actuators.accel, self.CCP.ACCEL_MIN, self.CCP.ACCEL_MAX) if CC.longActive else 0
       stopping = actuators.longControlState == LongCtrlState.stopping
-      starting = actuators.longControlState == LongCtrlState.starting
+      starting = actuators.longControlState == LongCtrlState.pid and (CS.esp_hold_confirmation or CS.out.vEgo < self.CP.vEgoStopping)
       can_sends.extend(self.CCS.create_acc_accel_control(self.packer_pt, CANBUS.pt, CS.acc_type, CC.longActive, accel,
                                                          acc_control, stopping, starting, CS.esp_hold_confirmation))
 

selfdrive/car/volkswagen/carstate.py

@@ -4,7 +4,7 @@ from openpilot.common.conversions import Conversions as CV
 from openpilot.selfdrive.car.interfaces import CarStateBase
 from opendbc.can.parser import CANParser
 from openpilot.selfdrive.car.volkswagen.values import DBC, CANBUS, PQ_CARS, NetworkLocation, TransmissionType, GearShifter, \
-                                            CarControllerParams
+                                            CarControllerParams, VolkswagenFlags
 
 
 class CarState(CarStateBase):
@@ -14,6 +14,7 @@ class CarState(CarStateBase):
     self.button_states = {button.event_type: False for button in self.CCP.BUTTONS}
     self.esp_hold_confirmation = False
     self.upscale_lead_car_signal = False
+    self.eps_stock_values = False
 
   def create_button_events(self, pt_cp, buttons):
     button_events = []
@@ -59,6 +60,11 @@ class CarState(CarStateBase):
     ret.steerFaultPermanent = hca_status in ("DISABLED", "FAULT")
     ret.steerFaultTemporary = hca_status in ("INITIALIZING", "REJECTED")
 
+    # VW Emergency Assist status tracking and mitigation
+    self.eps_stock_values = pt_cp.vl["LH_EPS_03"]
+    if self.CP.flags & VolkswagenFlags.STOCK_HCA_PRESENT:
+      ret.carFaultedNonCritical = bool(cam_cp.vl["HCA_01"]["EA_Ruckfreigabe"]) or cam_cp.vl["HCA_01"]["EA_ACC_Sollstatus"] > 0
+
     # Update gas, brakes, and gearshift.
     ret.gas = pt_cp.vl["Motor_20"]["MO_Fahrpedalrohwert_01"] / 100.0
     ret.gasPressed = ret.gas > 0
@@ -293,6 +299,11 @@ class CarState(CarStateBase):
 
     messages = []
 
+    if CP.flags & VolkswagenFlags.STOCK_HCA_PRESENT:
+      messages += [
+        ("HCA_01", 1),  # From R242 Driver assistance camera, 50Hz if steering/1Hz if not
+      ]
+
     if CP.networkLocation == NetworkLocation.fwdCamera:
       messages += [
         # sig_address, frequency

selfdrive/car/volkswagen/fingerprints.py

@@ -540,6 +540,7 @@ FW_VERSIONS = {
       b'\xf1\x875NA907115E \xf1\x890003',
       b'\xf1\x875NA907115E \xf1\x890005',
       b'\xf1\x875NA907115J \xf1\x890002',
+      b'\xf1\x875NA907115K \xf1\x890004',
       b'\xf1\x8783A907115  \xf1\x890007',
       b'\xf1\x8783A907115B \xf1\x890005',
       b'\xf1\x8783A907115F \xf1\x890002',
@@ -570,6 +571,7 @@ FW_VERSIONS = {
       b'\xf1\x870GC300046Q \xf1\x892802',
     ],
     (Ecu.srs, 0x715, None): [
+      b'\xf1\x875Q0959655AG\xf1\x890336\xf1\x82\x1316143231313500314617011730179333423100',
       b'\xf1\x875Q0959655AG\xf1\x890338\xf1\x82\x1316143231313500314617011730179333423100',
       b'\xf1\x875Q0959655AR\xf1\x890317\xf1\x82\x1331310031333334313132573732379333313100',
       b'\xf1\x875Q0959655BJ\xf1\x890336\xf1\x82\x1312110031333300314232583732379333423100',
@@ -587,6 +589,7 @@ FW_VERSIONS = {
     (Ecu.eps, 0x712, None): [
       b'\xf1\x875Q0909143M \xf1\x892041\xf1\x820529A6060603',
       b'\xf1\x875Q0909143P \xf1\x892051\xf1\x820527A6050705',
+      b'\xf1\x875Q0909143P \xf1\x892051\xf1\x820527A6070705',
       b'\xf1\x875Q0909144AB\xf1\x891082\xf1\x82\x0521A60604A1',
       b'\xf1\x875Q0910143C \xf1\x892211\xf1\x82\x0567A6000600',
       b'\xf1\x875Q0910143C \xf1\x892211\xf1\x82\x0567A6017A00',

selfdrive/car/volkswagen/interface.py

@@ -3,7 +3,7 @@ from panda import Panda
 from openpilot.common.conversions import Conversions as CV
 from openpilot.selfdrive.car import get_safety_config
 from openpilot.selfdrive.car.interfaces import CarInterfaceBase
-from openpilot.selfdrive.car.volkswagen.values import CAR, PQ_CARS, CANBUS, NetworkLocation, TransmissionType, GearShifter
+from openpilot.selfdrive.car.volkswagen.values import CAR, PQ_CARS, CANBUS, NetworkLocation, TransmissionType, GearShifter, VolkswagenFlags
 
 ButtonType = car.CarState.ButtonEvent.Type
 EventName = car.CarEvent.EventName
@@ -67,6 +67,9 @@ class CarInterface(CarInterfaceBase):
       else:
         ret.networkLocation = NetworkLocation.fwdCamera
 
+      if 0x126 in fingerprint[2]:  # HCA_01
+        ret.flags |= VolkswagenFlags.STOCK_HCA_PRESENT.value
+
     # Global lateral tuning defaults, can be overridden per-vehicle
 
     ret.steerActuatorDelay = 0.1
@@ -90,11 +93,9 @@ class CarInterface(CarInterfaceBase):
 
     ret.pcmCruise = not ret.openpilotLongitudinalControl
     ret.stoppingControl = True
-    ret.startingState = True
-    ret.startAccel = 1.0
     ret.stopAccel = -0.55
-    ret.vEgoStarting = 1.0
+    ret.vEgoStarting = 0.1
-    ret.vEgoStopping = 1.0
+    ret.vEgoStopping = 0.5
     ret.longitudinalTuning.kpV = [0.1]
     ret.longitudinalTuning.kiV = [0.0]
 

selfdrive/car/volkswagen/mqbcan.py

@@ -10,6 +10,24 @@ def create_steering_control(packer, bus, apply_steer, lkas_enabled):
   return packer.make_can_msg("HCA_01", bus, values)
 
 
+def create_eps_update(packer, bus, eps_stock_values, ea_simulated_torque):
+  values = {s: eps_stock_values[s] for s in [
+    "COUNTER",                     # Sync counter value to EPS output
+    "EPS_Lenkungstyp",             # EPS rack type
+    "EPS_Berechneter_LW",          # Absolute raw steering angle
+    "EPS_VZ_BLW",                  # Raw steering angle sign
+    "EPS_HCA_Status",              # EPS HCA control status
+  ]}
+
+  values.update({
+    # Absolute driver torque input and sign, with EA inactivity mitigation
+    "EPS_Lenkmoment": abs(ea_simulated_torque),
+    "EPS_VZ_Lenkmoment": 1 if ea_simulated_torque < 0 else 0,
+  })
+
+  return packer.make_can_msg("LH_EPS_03", bus, values)
+
+
 def create_lka_hud_control(packer, bus, ldw_stock_values, enabled, steering_pressed, hud_alert, hud_control):
   values = {}
   if len(ldw_stock_values):
@@ -94,7 +112,7 @@ def create_acc_accel_control(packer, bus, acc_type, acc_enabled, accel, acc_cont
     acc_hold_type = 0
 
   acc_07_values = {
-    "ACC_Anhalteweg": 0.75 if stopping else 20.46,  # Distance to stop (stopping coordinator handles terminal roll-out)
+    "ACC_Anhalteweg": 0.3 if stopping else 20.46,  # Distance to stop (stopping coordinator handles terminal roll-out)
     "ACC_Freilauf_Info": 2 if acc_enabled else 0,
     "ACC_Folgebeschl": 3.02,  # Not using secondary controller accel unless and until we understand its impact
     "ACC_Sollbeschleunigung_02": accel if acc_enabled else 3.01,

selfdrive/car/volkswagen/values.py

@@ -1,6 +1,6 @@
 from collections import defaultdict, namedtuple
 from dataclasses import dataclass, field
-from enum import Enum, StrEnum
+from enum import Enum, IntFlag, StrEnum
 from typing import Dict, List, Union
 
 from cereal import car
@@ -109,6 +109,10 @@ class CANBUS:
   cam = 2
 
 
+class VolkswagenFlags(IntFlag):
+  STOCK_HCA_PRESENT = 1
+
+
 # Check the 7th and 8th characters of the VIN before adding a new CAR. If the
 # chassis code is already listed below, don't add a new CAR, just add to the
 # FW_VERSIONS for that existing CAR.

selfdrive/controls/controlsd.py

@@ -8,7 +8,6 @@ from typing import SupportsFloat
 from cereal import car, log
 from openpilot.common.numpy_fast import clip
 from openpilot.common.realtime import config_realtime_process, Priority, Ratekeeper, DT_CTRL
-from openpilot.common.profiler import Profiler
 from openpilot.common.params import Params
 import cereal.messaging as messaging
 from cereal.visionipc import VisionIpcClient, VisionStreamType
@@ -195,7 +194,6 @@ class Controls:
 
     # controlsd is driven by can recv, expected at 100Hz
     self.rk = Ratekeeper(100, print_delay_threshold=None)
-    self.prof = Profiler(False)  # off by default
 
   def set_initial_state(self):
     if REPLAY:
@@ -851,12 +849,10 @@ class Controls:
 
   def step(self):
     start_time = time.monotonic()
-    self.prof.checkpoint("Ratekeeper", ignore=True)
 
     # Sample data from sockets and get a carState
     CS = self.data_sample()
     cloudlog.timestamp("Data sampled")
-    self.prof.checkpoint("Sample")
 
     self.update_events(CS)
     cloudlog.timestamp("Events updated")
@@ -864,16 +860,12 @@ class Controls:
     if not self.CP.passive and self.initialized:
       # Update control state
       self.state_transition(CS)
-      self.prof.checkpoint("State transition")
 
     # Compute actuators (runs PID loops and lateral MPC)
     CC, lac_log = self.state_control(CS)
 
-    self.prof.checkpoint("State Control")
-
     # Publish data
     self.publish_logs(CS, start_time, CC, lac_log)
-    self.prof.checkpoint("Sent")
 
     self.CS_prev = CS
 
@@ -893,7 +885,6 @@ class Controls:
       while True:
         self.step()
         self.rk.monitor_time()
-        self.prof.display()
     except SystemExit:
       e.set()
       t.join()

selfdrive/controls/lib/longitudinal_planner.py

@@ -47,13 +47,14 @@ def limit_accel_in_turns(v_ego, angle_steers, a_target, CP):
 
 
 class LongitudinalPlanner:
-  def __init__(self, CP, init_v=0.0, init_a=0.0):
+  def __init__(self, CP, init_v=0.0, init_a=0.0, dt=DT_MDL):
     self.CP = CP
     self.mpc = LongitudinalMpc()
     self.fcw = False
+    self.dt = dt
 
     self.a_desired = init_a
-    self.v_desired_filter = FirstOrderFilter(init_v, 2.0, DT_MDL)
+    self.v_desired_filter = FirstOrderFilter(init_v, 2.0, self.dt)
     self.v_model_error = 0.0
 
     self.v_desired_trajectory = np.zeros(CONTROL_N)
@@ -148,8 +149,8 @@ class LongitudinalPlanner:
 
     # Interpolate 0.05 seconds and save as starting point for next iteration
     a_prev = self.a_desired
-    self.a_desired = float(interp(DT_MDL, ModelConstants.T_IDXS[:CONTROL_N], self.a_desired_trajectory))
+    self.a_desired = float(interp(self.dt, ModelConstants.T_IDXS[:CONTROL_N], self.a_desired_trajectory))
-    self.v_desired_filter.x = self.v_desired_filter.x + DT_MDL * (self.a_desired + a_prev) / 2.0
+    self.v_desired_filter.x = self.v_desired_filter.x + self.dt * (self.a_desired + a_prev) / 2.0
 
   def publish(self, sm, pm):
     plan_send = messaging.new_message('longitudinalPlan')

selfdrive/controls/tests/test_startup.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-import time
+import os
 import unittest
 from parameterized import parameterized
 
@@ -11,7 +11,7 @@ from openpilot.selfdrive.car.fingerprints import _FINGERPRINTS
 from openpilot.selfdrive.car.toyota.values import CAR as TOYOTA
 from openpilot.selfdrive.car.mazda.values import CAR as MAZDA
 from openpilot.selfdrive.controls.lib.events import EVENT_NAME
-from openpilot.selfdrive.test.helpers import with_processes
+from openpilot.selfdrive.manager.process_config import managed_processes
 
 EventName = car.CarEvent.EventName
 Ecu = car.CarParams.Ecu
@@ -38,6 +38,9 @@ CX5_FW_VERSIONS = [
 
 class TestStartup(unittest.TestCase):
 
+  def tearDown(self):
+    managed_processes['controlsd'].stop()
+
   @parameterized.expand([
     # TODO: test EventName.startup for release branches
 
@@ -61,15 +64,11 @@ class TestStartup(unittest.TestCase):
     (EventName.startupMaster, TOYOTA.COROLLA, COROLLA_FW_VERSIONS_FUZZY, "toyota"),
     (EventName.startupMaster, TOYOTA.COROLLA, COROLLA_FW_VERSIONS_FUZZY, "toyota"),
   ])
-  @with_processes(['controlsd'])
   def test_startup_alert(self, expected_event, car_model, fw_versions, brand):
-
-    # TODO: this should be done without any real sockets
     controls_sock = messaging.sub_sock("controlsState")
     pm = messaging.PubMaster(['can', 'pandaStates'])
 
     params = Params()
-    params.clear_all()
     params.put_bool("Passive", False)
     params.put_bool("OpenpilotEnabledToggle", True)
 
@@ -91,11 +90,13 @@ class TestStartup(unittest.TestCase):
       cp.carVin = "1" * 17
       cp.carFw = car_fw
       params.put("CarParamsCache", cp.to_bytes())
+    else:
+      os.environ['SKIP_FW_QUERY'] = '1'
 
-    time.sleep(2) # wait for controlsd to be ready
+    managed_processes['controlsd'].start()
 
+    assert pm.wait_for_readers_to_update('can', 5)
     pm.send('can', can_list_to_can_capnp([[0, 0, b"", 0]]))
-    time.sleep(0.1)
 
     msg = messaging.new_message('pandaStates', 1)
     msg.pandaStates[0].pandaType = log.PandaState.PandaType.uno
@@ -107,18 +108,18 @@ class TestStartup(unittest.TestCase):
     else:
       finger = _FINGERPRINTS[car_model][0]
 
+    msgs = [[addr, 0, b'\x00'*length, 0] for addr, length in finger.items()]
     for _ in range(1000):
       # controlsd waits for boardd to echo back that it has changed the multiplexing mode
       if not params.get_bool("ObdMultiplexingChanged"):
         params.put_bool("ObdMultiplexingChanged", True)
 
-      msgs = [[addr, 0, b'\x00'*length, 0] for addr, length in finger.items()]
       pm.send('can', can_list_to_can_capnp(msgs))
+      assert pm.wait_for_readers_to_update('can', 2, dt=0.001)
 
-      time.sleep(0.01)
+      ctrls = messaging.drain_sock(controls_sock)
-      msgs = messaging.drain_sock(controls_sock)
+      if len(ctrls):
-      if len(msgs):
+        event_name = ctrls[0].controlsState.alertType.split("/")[0]
-        event_name = msgs[0].controlsState.alertType.split("/")[0]
         self.assertEqual(EVENT_NAME[expected_event], event_name,
                          f"expected {EVENT_NAME[expected_event]} for '{car_model}', got {event_name}")
         break

selfdrive/debug/can_print_changes.py

@@ -3,10 +3,11 @@ import argparse
 import binascii
 import time
 from collections import defaultdict
+from typing import Optional
 
 import cereal.messaging as messaging
 from openpilot.selfdrive.debug.can_table import can_table
-from openpilot.tools.lib.logreader import logreader_from_route_or_segment
+from openpilot.tools.lib.logreader import LogIterable, LogReader
 
 RED = '\033[91m'
 CLEAR = '\033[0m'
@@ -95,13 +96,15 @@ if __name__ == "__main__":
 
   args = parser.parse_args()
 
-  init_lr, new_lr = None, None
+  init_lr: Optional[LogIterable] = None
+  new_lr: Optional[LogIterable] = None
+
   if args.init:
     if args.init == '':
       init_lr = []
     else:
-      init_lr = logreader_from_route_or_segment(args.init)
+      init_lr = LogReader(args.init)
   if args.comp:
-    new_lr = logreader_from_route_or_segment(args.comp)
+    new_lr = LogReader(args.comp)
 
   can_printer(args.bus, init_msgs=init_lr, new_msgs=new_lr, table=args.table)

selfdrive/debug/count_events.py

@@ -4,16 +4,12 @@ import math
 import datetime
 from collections import Counter
 from pprint import pprint
-from tqdm import tqdm
 from typing import List, Tuple, cast
 
 from cereal.services import SERVICE_LIST
-from openpilot.tools.lib.route import Route
+from openpilot.tools.lib.logreader import LogReader, ReadMode
-from openpilot.tools.lib.logreader import LogReader
 
 if __name__ == "__main__":
-  r = Route(sys.argv[1])
-
   cnt_valid: Counter = Counter()
   cnt_events: Counter = Counter()
 
@@ -24,31 +20,29 @@ if __name__ == "__main__":
   start_time = math.inf
   end_time = -math.inf
   ignition_off = None
-  for q in tqdm(r.qlog_paths()):
+  for msg in LogReader(sys.argv[1], ReadMode.QLOG):
-    if q is None:
+    end_time = max(end_time, msg.logMonoTime)
-      continue
+    start_time = min(start_time, msg.logMonoTime)
-    lr = list(LogReader(q))
-    for msg in lr:
-      end_time = max(end_time, msg.logMonoTime)
-      start_time = min(start_time, msg.logMonoTime)
 
-      if msg.which() == 'onroadEvents':
+    if msg.which() == 'onroadEvents':
-        for e in msg.onroadEvents:
+      for e in msg.onroadEvents:
-          cnt_events[e.name] += 1
+        cnt_events[e.name] += 1
-      elif msg.which() == 'controlsState':
+    elif msg.which() == 'controlsState':
-        if len(alerts) == 0 or alerts[-1][1] != msg.controlsState.alertType:
+      at = msg.controlsState.alertType
+      if "/override" not in at or "lanechange" in at.lower():
+        if len(alerts) == 0 or alerts[-1][1] != at:
           t = (msg.logMonoTime - start_time) / 1e9
-          alerts.append((t, msg.controlsState.alertType))
+          alerts.append((t, at))
-      elif msg.which() == 'pandaStates':
+    elif msg.which() == 'pandaStates':
-        if ignition_off is None:
+      if ignition_off is None:
-          ign = any(ps.ignitionLine or ps.ignitionCan for ps in msg.pandaStates)
+        ign = any(ps.ignitionLine or ps.ignitionCan for ps in msg.pandaStates)
-          if not ign:
+        if not ign:
-            ignition_off = msg.logMonoTime
+          ignition_off = msg.logMonoTime
-      elif msg.which() in cams:
+    elif msg.which() in cams:
-        cnt_cameras[msg.which()] += 1
+      cnt_cameras[msg.which()] += 1
 
-      if not msg.valid:
+    if not msg.valid:
-        cnt_valid[msg.which()] += 1
+      cnt_valid[msg.which()] += 1
 
   duration = (end_time - start_time) / 1e9
 

selfdrive/debug/filter_log_message.py

@@ -1,11 +1,9 @@
 #!/usr/bin/env python3
-import os
 import argparse
 import json
 
 import cereal.messaging as messaging
 from openpilot.tools.lib.logreader import LogReader
-from openpilot.tools.lib.route import Route
 
 LEVELS = {
   "DEBUG": 10,
@@ -53,31 +51,18 @@ if __name__ == "__main__":
   parser.add_argument("route", type=str, nargs='*', help="route name + segment number for offline usage")
   args = parser.parse_args()
 
-  logs = None
-  if len(args.route):
-    if os.path.exists(args.route[0]):
-      logs = [args.route[0]]
-    else:
-      r = Route(args.route[0])
-      logs = [q_log if r_log is None else r_log for (q_log, r_log) in zip(r.qlog_paths(), r.log_paths(), strict=True)]
-
-  if len(args.route) == 2 and logs:
-    n = int(args.route[1])
-    logs = [logs[n]]
-
   min_level = LEVELS[args.level]
 
-  if logs:
+  if args.route:
-    for log in logs:
+    for route in args.route:
-      if log:
+      lr = LogReader(route)
-        lr = LogReader(log)
+      for m in lr:
-        for m in lr:
+        if m.which() == 'logMessage':
-          if m.which() == 'logMessage':
+          print_logmessage(m.logMonoTime, m.logMessage, min_level)
-            print_logmessage(m.logMonoTime, m.logMessage, min_level)
+        elif m.which() == 'errorLogMessage':
-          elif m.which() == 'errorLogMessage' and 'qlog' in log:
+          print_logmessage(m.logMonoTime, m.errorLogMessage, min_level)
-            print_logmessage(m.logMonoTime, m.errorLogMessage, min_level)
+        elif m.which() == 'androidLog':
-          elif m.which() == 'androidLog':
+          print_androidlog(m.logMonoTime, m.androidLog)
-            print_androidlog(m.logMonoTime, m.androidLog)
   else:
     sm = messaging.SubMaster(['logMessage', 'androidLog'], addr=args.addr)
     while True:

selfdrive/debug/fingerprint_from_route.py

@@ -1,8 +1,7 @@
 #!/usr/bin/env python3
 
 import sys
-from openpilot.tools.lib.route import Route
+from openpilot.tools.lib.logreader import LogReader, ReadMode
-from openpilot.tools.lib.logreader import MultiLogIterator
 
 
 def get_fingerprint(lr):
@@ -40,6 +39,5 @@ if __name__ == "__main__":
     print("Usage: ./fingerprint_from_route.py <route>")
     sys.exit(1)
 
-  route = Route(sys.argv[1])
+  lr = LogReader(sys.argv[1], ReadMode.QLOG)
-  lr = MultiLogIterator(route.log_paths()[:5])
   get_fingerprint(lr)

selfdrive/debug/print_docs_diff.py

@@ -8,9 +8,10 @@ from openpilot.selfdrive.car.docs import get_all_car_info
 from openpilot.selfdrive.car.docs_definitions import Column
 
 FOOTNOTE_TAG = "<sup>{}</sup>"
-STAR_ICON = '<a href="##"><img valign="top" src="https://raw.githubusercontent.com/commaai/openpilot/master/docs/assets/icon-star-{}.svg" width="22" /></a>'
+STAR_ICON = '<a href="##"><img valign="top" ' + \
-VIDEO_ICON = '<a href="{}" target="_blank">\
+            'src="https://media.githubusercontent.com/media/commaai/openpilot/master/docs/assets/icon-star-{}.svg" width="22" /></a>'
-                          <img height="18px" src="https://raw.githubusercontent.com/commaai/openpilot/master/docs/assets/icon-youtube.svg"></img></a>'
+VIDEO_ICON = '<a href="{}" target="_blank">' + \
+             '<img height="18px" src="https://media.githubusercontent.com/media/commaai/openpilot/master/docs/assets/icon-youtube.svg"></img></a>'
 COLUMNS = "|" + "|".join([column.value for column in Column]) + "|"
 COLUMN_HEADER = "|---|---|---|{}|".format("|".join([":---:"] * (len(Column) - 3)))
 ARROW_SYMBOL = "➡️"

selfdrive/debug/run_process_on_route.py

@@ -3,14 +3,12 @@
 import argparse
 
 from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, replay_process
-from openpilot.tools.lib.logreader import MultiLogIterator
-from openpilot.tools.lib.route import Route
 from openpilot.tools.lib.helpers import save_log
+from openpilot.tools.lib.logreader import LogReader
 
 if __name__ == "__main__":
   parser = argparse.ArgumentParser(description="Run process on route and create new logs",
                                    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-  parser.add_argument("--qlog", help="Use qlog instead of log", action="store_true")
   parser.add_argument("--fingerprint", help="The fingerprint to use")
   parser.add_argument("route", help="The route name to use")
   parser.add_argument("process", help="The process to run")
@@ -18,8 +16,7 @@ if __name__ == "__main__":
 
   cfg = [c for c in CONFIGS if c.proc_name == args.process][0]
 
-  route = Route(args.route)
+  lr = LogReader(args.route)
-  lr = MultiLogIterator(route.qlog_paths() if args.qlog else route.log_paths())
   inputs = list(lr)
 
   outputs = replay_process(cfg, inputs, fingerprint=args.fingerprint)
@@ -29,5 +26,5 @@ if __name__ == "__main__":
   inputs = [i for i in inputs if i.which() not in produces]
   outputs = sorted(inputs + outputs, key=lambda x: x.logMonoTime)
 
-  fn = f"{args.route}_{args.process}.bz2"
+  fn = f"{args.route.replace('/', '_')}_{args.process}.bz2"
   save_log(fn, outputs)

selfdrive/debug/sensor_data_to_hist.py

@@ -1,111 +0,0 @@
-#!/usr/bin/env python3
-'''
-printing the gap between interrupts in a histogram to check if the
-frequency is what we expect, the bmx is not interrupt driven for as we
-get interrupts in a 2kHz rate.
-'''
-
-import argparse
-import sys
-import numpy as np
-from collections import defaultdict
-
-from openpilot.tools.lib.logreader import LogReader
-from openpilot.tools.lib.route import Route
-
-import matplotlib.pyplot as plt
-
-SRC_BMX = "bmx055"
-SRC_LSM = "lsm6ds3"
-
-
-def parseEvents(log_reader):
-  bmx_data = defaultdict(list)
-  lsm_data = defaultdict(list)
-
-  for m in log_reader:
-    if m.which() not in ['accelerometer', 'gyroscope']:
-      continue
-
-    d = getattr(m, m.which()).to_dict()
-
-    if d["source"] == SRC_BMX and "acceleration" in d:
-      bmx_data["accel"].append(d["timestamp"] / 1e9)
-
-    if d["source"] == SRC_BMX and "gyroUncalibrated" in d:
-      bmx_data["gyro"].append(d["timestamp"] / 1e9)
-
-    if d["source"] == SRC_LSM and "acceleration" in d:
-      lsm_data["accel"].append(d["timestamp"] / 1e9)
-
-    if d["source"] == SRC_LSM and "gyroUncalibrated" in d:
-      lsm_data["gyro"].append(d["timestamp"] / 1e9)
-
-  return bmx_data, lsm_data
-
-
-def cleanData(data):
-  if len(data) == 0:
-    return [], []
-
-  data.sort()
-  diffs = np.diff(data)
-  return data, diffs
-
-
-def logAvgValues(data, sensor):
-  if len(data) == 0:
-    print(f"{sensor}: no data to average")
-    return
-
-  avg = sum(data) / len(data)
-  hz  = 1 / avg
-  print(f"{sensor}: data_points: {len(data)} avg [ns]: {avg} avg [Hz]: {hz}")
-
-
-if __name__ == "__main__":
-  parser = argparse.ArgumentParser()
-  parser.add_argument("route", type=str, help="route name")
-  parser.add_argument("segment", type=int, help="segment number")
-  args = parser.parse_args()
-
-  r = Route(args.route)
-  logs = r.log_paths()
-
-  if len(logs) == 0:
-    print("NO data routes")
-    sys.exit(0)
-
-  if args.segment >= len(logs):
-    print(f"RouteID: {args.segment} out of range, max: {len(logs) -1}")
-    sys.exit(0)
-
-  lr = LogReader(logs[args.segment])
-  bmx_data, lsm_data = parseEvents(lr)
-
-  # sort bmx accel data, and then cal all the diffs, and to a histogram of those
-  bmx_accel, bmx_accel_diffs = cleanData(bmx_data["accel"])
-  bmx_gyro, bmx_gyro_diffs = cleanData(bmx_data["gyro"])
-  lsm_accel, lsm_accel_diffs = cleanData(lsm_data["accel"])
-  lsm_gyro, lsm_gyro_diffs = cleanData(lsm_data["gyro"])
-
-  # get out the averages
-  logAvgValues(bmx_accel_diffs, "bmx accel")
-  logAvgValues(bmx_gyro_diffs, "bmx gyro ")
-  logAvgValues(lsm_accel_diffs, "lsm accel")
-  logAvgValues(lsm_gyro_diffs, "lsm gyro ")
-
-  fig, axs = plt.subplots(1, 2, tight_layout=True)
-  axs[0].hist(bmx_accel_diffs, bins=50)
-  axs[0].set_title("bmx_accel")
-  axs[1].hist(bmx_gyro_diffs, bins=50)
-  axs[1].set_title("bmx_gyro")
-
-  figl, axsl = plt.subplots(1, 2, tight_layout=True)
-  axsl[0].hist(lsm_accel_diffs, bins=50)
-  axsl[0].set_title("lsm_accel")
-  axsl[1].hist(lsm_gyro_diffs, bins=50)
-  axsl[1].set_title("lsm_gyro")
-
-  print("check plot...")
-  plt.show()

selfdrive/debug/test_fw_query_on_routes.py

@@ -112,8 +112,8 @@ if __name__ == "__main__":
           padding = max([len(fw.brand or UNKNOWN_BRAND) for fw in car_fw])
           for version in sorted(car_fw, key=lambda fw: fw.brand):
             subaddr = None if version.subAddress == 0 else hex(version.subAddress)
-            print(f"  Brand: {version.brand or UNKNOWN_BRAND:{padding}}, bus: {version.bus} - \
+            print(f"  Brand: {version.brand or UNKNOWN_BRAND:{padding}}, bus: {version.bus} - " +
-                      (Ecu.{version.ecu}, {hex(version.address)}, {subaddr}): [{version.fwVersion}],")
+                  "(Ecu.{version.ecu}, {hex(version.address)}, {subaddr}): [{version.fwVersion}],")
 
           print("Mismatches")
           found = False

selfdrive/debug/toyota_eps_factor.py

@@ -5,8 +5,7 @@ import matplotlib.pyplot as plt
 from sklearn import linear_model
 from openpilot.selfdrive.car.toyota.values import STEER_THRESHOLD
 
-from openpilot.tools.lib.route import Route
+from openpilot.tools.lib.logreader import LogReader
-from openpilot.tools.lib.logreader import MultiLogIterator
 
 MIN_SAMPLES = 30 * 100
 
@@ -58,7 +57,6 @@ def get_eps_factor(lr, plot=False):
 
 
 if __name__ == "__main__":
-  r = Route(sys.argv[1])
+  lr = LogReader(sys.argv[1])
-  lr = MultiLogIterator(r.log_paths())
   n = get_eps_factor(lr, plot="--plot" in sys.argv)
   print("EPS torque factor: ", n)

selfdrive/locationd/test/test_locationd_scenarios.py

@@ -5,12 +5,10 @@ import numpy as np
 from collections import defaultdict
 from enum import Enum
 
-
-from openpilot.selfdrive.test.openpilotci import get_url
 from openpilot.tools.lib.logreader import LogReader
 from openpilot.selfdrive.test.process_replay.process_replay import replay_process_with_name
 
-TEST_ROUTE, TEST_SEG_NUM = "ff2bd20623fcaeaa|2023-09-05--10-14-54", 4
+TEST_ROUTE = "ff2bd20623fcaeaa|2023-09-05--10-14-54/4"
 GPS_MESSAGES = ['gpsLocationExternal', 'gpsLocation']
 SELECT_COMPARE_FIELDS = {
   'yaw_rate': ['angularVelocityCalibrated', 'value', 2],
@@ -108,7 +106,7 @@ class TestLocationdScenarios(unittest.TestCase):
 
   @classmethod
   def setUpClass(cls):
-    cls.logs = list(LogReader(get_url(TEST_ROUTE, TEST_SEG_NUM)))
+    cls.logs = list(LogReader(TEST_ROUTE))
 
   def test_base(self):
     """

selfdrive/manager/manager.py

@@ -229,6 +229,8 @@ if __name__ == "__main__":
 
   try:
     main()
+  except KeyboardInterrupt:
+    print("got CTRL-C, exiting")
   except Exception:
     add_file_handler(cloudlog)
     cloudlog.exception("Manager failed to start")

selfdrive/modeld/models/navmodel.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:adc5aca6753b6ae0a1469f3e5bcb943d00cc9de75218489f2e4c3d960e7af048
+oid sha256:4971931accb5ba2e534bb3e0c591826ee507e2988df2eccf1fe862c303ddf9c5
-size 14138061
+size 14221074

selfdrive/modeld/models/navmodel_q.dlc

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c808717d073a0bb347f9ba929953c0b2b792ce9997f343f7e44a0b2b0e139132
+oid sha256:fa346ada6f8c6326a5ee5fcd27e45e3e710049358079413c6a4624b20c6e1e47
 size 3630942

selfdrive/modeld/models/supercombo.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cf6133c5bff295a3ee69eeb01297ba77adb6b83dbc1d774442a48117dbaf4626
+oid sha256:ae44fe832fe48b89998f09cebb1bcd129864a8f51497b636cd38e66e46d69a89
-size 48457192
+size 48457850

selfdrive/navd/navd.py

@@ -71,8 +71,11 @@ class RouteEngine:
         self.ui_pid = ui_pid[0]
 
     self.update_location()
-    self.recompute_route()
+    try:
-    self.send_instruction()
+      self.recompute_route()
+      self.send_instruction()
+    except Exception:
+      cloudlog.exception("navd.failed_to_compute")
 
   def update_location(self):
     location = self.sm['liveLocationKalman']
@@ -256,7 +259,10 @@ class RouteEngine:
     for i in range(self.step_idx + 1, len(self.route)):
       total_distance += self.route[i]['distance']
       total_time += self.route[i]['duration']
-      total_time_typical += self.route[i]['duration_typical']
+      if self.route[i]['duration_typical'] is None:
+        total_time_typical += self.route[i]['duration']
+      else:
+        total_time_typical += self.route[i]['duration_typical']
 
     msg.navInstruction.distanceRemaining = total_distance
     msg.navInstruction.timeRemaining = total_time

selfdrive/navd/tests/test_navd.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+import json
+import random
+import unittest
+import numpy as np
+
+import cereal.messaging as messaging
+from openpilot.common.params import Params
+from openpilot.selfdrive.manager.process_config import managed_processes
+
+
+class TestNavd(unittest.TestCase):
+  def setUp(self):
+    self.params = Params()
+    self.sm = messaging.SubMaster(['navRoute', 'navInstruction'])
+
+  def tearDown(self):
+    managed_processes['navd'].stop()
+
+  def _check_route(self, start, end, check_coords=True):
+    self.params.put("NavDestination", json.dumps(end))
+    self.params.put("LastGPSPosition", json.dumps(start))
+
+    managed_processes['navd'].start()
+    for _ in range(30):
+      self.sm.update(1000)
+      if all(f > 0 for f in self.sm.rcv_frame.values()):
+        break
+    else:
+      raise Exception("didn't get a route")
+
+    assert managed_processes['navd'].proc.is_alive()
+    managed_processes['navd'].stop()
+
+    # ensure start and end match up
+    if check_coords:
+      coords = self.sm['navRoute'].coordinates
+      assert np.allclose([start['latitude'], start['longitude'], end['latitude'], end['longitude']],
+                         [coords[0].latitude, coords[0].longitude, coords[-1].latitude, coords[-1].longitude],
+                         rtol=1e-3)
+
+  def test_simple(self):
+    start = {
+      "latitude": 32.7427228,
+      "longitude": -117.2321177,
+    }
+    end = {
+      "latitude": 32.7557004,
+      "longitude": -117.268002,
+    }
+    self._check_route(start, end)
+
+  def test_random(self):
+    for _ in range(10):
+      start = {"latitude": random.uniform(-90, 90), "longitude": random.uniform(-180, 180)}
+      end = {"latitude": random.uniform(-90, 90), "longitude": random.uniform(-180, 180)}
+      self._check_route(start, end, check_coords=False)
+
+
+if __name__ == "__main__":
+  unittest.main()

selfdrive/test/helpers.py

@@ -8,6 +8,7 @@ from openpilot.common.params import Params
 from openpilot.selfdrive.manager.process_config import managed_processes
 from openpilot.system.hardware import PC
 from openpilot.system.version import training_version, terms_version
+from openpilot.tools.lib.logreader import LogIterable
 
 
 def set_params_enabled():
@@ -72,3 +73,18 @@ def with_processes(processes, init_time=0, ignore_stopped=None):
 
 def noop(*args, **kwargs):
   pass
+
+
+def read_segment_list(segment_list_path):
+  with open(segment_list_path, "r") as f:
+    seg_list = f.read().splitlines()
+
+  return [(platform[2:], segment) for platform, segment in zip(seg_list[::2], seg_list[1::2], strict=True)]
+
+
+# Utilities for sanitizing routes of only essential data for testing car ports and doing validation.
+
+PRESERVE_SERVICES = ["can", "carParams", "pandaStates", "pandaStateDEPRECATED"]
+
+def sanitize(lr: LogIterable) -> LogIterable:
+  return filter(lambda msg: msg.which() in PRESERVE_SERVICES, lr)

selfdrive/test/process_replay/model_replay.py

@@ -9,7 +9,7 @@ import cereal.messaging as messaging
 from openpilot.common.params import Params
 from openpilot.system.hardware import PC
 from openpilot.selfdrive.manager.process_config import managed_processes
-from openpilot.selfdrive.test.openpilotci import BASE_URL, get_url
+from openpilot.tools.lib.openpilotci import BASE_URL, get_url
 from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs, format_diff
 from openpilot.selfdrive.test.process_replay.process_replay import get_process_config, replay_process
 from openpilot.system.version import get_commit
@@ -143,7 +143,7 @@ if __name__ == "__main__":
     import requests
     import threading
     import http.server
-    from openpilot.selfdrive.test.openpilotci import upload_bytes
+    from openpilot.tools.lib.openpilotci import upload_bytes
     os.environ['MAPS_HOST'] = 'http://localhost:5000'
 
     class HTTPRequestHandler(http.server.BaseHTTPRequestHandler):
@@ -229,7 +229,7 @@ if __name__ == "__main__":
 
   # upload new refs
   if (update or failed) and not PC:
-    from openpilot.selfdrive.test.openpilotci import upload_file
+    from openpilot.tools.lib.openpilotci import upload_file
 
     print("Uploading new refs")
 

selfdrive/test/process_replay/model_replay_ref_commit

@@ -1 +1 @@
-91cd2bf71771c2770c0effc26c0bb23d27208138
+ad64b6f38c1362e9d184f3fc95299284eacb56d4

selfdrive/test/process_replay/ref_commit

@@ -1 +1 @@
-ea96f935a7a16c53623c3b03e70c0fbfa6b249e7
+1b981ce7f817974d4a7a28b06f01f727a5a7ea7b

selfdrive/test/process_replay/test_debayer.py

@@ -8,14 +8,13 @@ import pyopencl as cl  # install with `PYOPENCL_CL_PRETEND_VERSION=2.0 pip insta
 
 from openpilot.system.hardware import PC, TICI
 from openpilot.common.basedir import BASEDIR
-from openpilot.selfdrive.test.openpilotci import BASE_URL, get_url
+from openpilot.tools.lib.openpilotci import BASE_URL
 from openpilot.system.version import get_commit
 from openpilot.system.camerad.snapshot.snapshot import yuv_to_rgb
 from openpilot.tools.lib.logreader import LogReader
 from openpilot.tools.lib.filereader import FileReader
 
-TEST_ROUTE = "8345e3b82948d454|2022-05-04--13-45-33"
+TEST_ROUTE = "8345e3b82948d454|2022-05-04--13-45-33/0"
-SEGMENT = 0
 
 FRAME_WIDTH = 1928
 FRAME_HEIGHT = 1208
@@ -116,7 +115,7 @@ if __name__ == "__main__":
   ref_commit_fn = os.path.join(replay_dir, "debayer_replay_ref_commit")
 
   # load logs
-  lr = list(LogReader(get_url(TEST_ROUTE, SEGMENT)))
+  lr = list(LogReader(TEST_ROUTE))
 
   # run replay
   frames = debayer_replay(lr)
@@ -172,7 +171,7 @@ if __name__ == "__main__":
 
   # upload new refs
   if update or (failed and TICI):
-    from openpilot.selfdrive.test.openpilotci import upload_file
+    from openpilot.tools.lib.openpilotci import upload_file
 
     print("Uploading new refs")
 

selfdrive/test/process_replay/test_fuzzy.py

@@ -14,13 +14,15 @@ import openpilot.selfdrive.test.process_replay.process_replay as pr
 # that openpilot makes causing error with NaN, inf, int size, array indexing ...
 # TODO: Make each one testable
 NOT_TESTED = ['controlsd', 'plannerd', 'calibrationd', 'dmonitoringd', 'paramsd', 'dmonitoringmodeld', 'modeld']
+
 TEST_CASES = [(cfg.proc_name, copy.deepcopy(cfg)) for cfg in pr.CONFIGS if cfg.proc_name not in NOT_TESTED]
 
 class TestFuzzProcesses(unittest.TestCase):
 
+  # TODO: make this faster and increase examples
   @parameterized.expand(TEST_CASES)
   @given(st.data())
-  @settings(phases=[Phase.generate, Phase.target], max_examples=50, deadline=1000, suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large])
+  @settings(phases=[Phase.generate, Phase.target], max_examples=10, deadline=1000, suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large])
   def test_fuzz_process(self, proc_name, cfg, data):
     msgs = FuzzyGenerator.get_random_event_msg(data.draw, events=cfg.pubs, real_floats=True)
     lr = [log.Event.new_message(**m).as_reader() for m in msgs]

selfdrive/test/process_replay/test_processes.py

@@ -8,7 +8,7 @@ from tqdm import tqdm
 from typing import Any, DefaultDict, Dict
 
 from openpilot.selfdrive.car.car_helpers import interface_names
-from openpilot.selfdrive.test.openpilotci import get_url, upload_file
+from openpilot.tools.lib.openpilotci import get_url, upload_file
 from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs, format_diff
 from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, PROC_REPLAY_DIR, FAKEDATA, check_openpilot_enabled, replay_process
 from openpilot.system.version import get_commit

selfdrive/test/process_replay/test_regen.py

@@ -6,7 +6,7 @@ from parameterized import parameterized
 
 from openpilot.selfdrive.test.process_replay.regen import regen_segment, DummyFrameReader
 from openpilot.selfdrive.test.process_replay.process_replay import check_openpilot_enabled
-from openpilot.selfdrive.test.openpilotci import get_url
+from openpilot.tools.lib.openpilotci import get_url
 from openpilot.tools.lib.logreader import LogReader
 from openpilot.tools.lib.framereader import FrameReader
 

selfdrive/test/test_onroad.py

@@ -34,7 +34,7 @@ PROCS = {
   "./encoderd": 17.0,
   "./camerad": 14.5,
   "./locationd": 11.0,
-  "./mapsd": (1.0, 10.0),
+  "./mapsd": (0.5, 10.0),
   "selfdrive.controls.plannerd": 11.0,
   "./ui": 18.0,
   "selfdrive.locationd.paramsd": 9.0,

selfdrive/test/test_valgrind_replay.py

@@ -15,7 +15,7 @@ else:
 import cereal.messaging as messaging
 from collections import namedtuple
 from openpilot.tools.lib.logreader import LogReader
-from openpilot.selfdrive.test.openpilotci import get_url
+from openpilot.tools.lib.openpilotci import get_url
 from openpilot.common.basedir import BASEDIR
 
 ProcessConfig = namedtuple('ProcessConfig', ['proc_name', 'pub_sub', 'ignore', 'command', 'path', 'segment', 'wait_for_response'])

selfdrive/test/update_ci_routes.py

@@ -11,7 +11,7 @@ from tqdm import tqdm
 
 from openpilot.selfdrive.car.tests.routes import routes as test_car_models_routes
 from openpilot.selfdrive.test.process_replay.test_processes import source_segments as replay_segments
-from openpilot.selfdrive.test.openpilotci import (DATA_CI_ACCOUNT, DATA_CI_ACCOUNT_URL, OPENPILOT_CI_CONTAINER,
+from openpilot.tools.lib.openpilotci import (DATA_CI_ACCOUNT, DATA_CI_ACCOUNT_URL, OPENPILOT_CI_CONTAINER,
                                                   DATA_CI_CONTAINER, get_azure_credential, get_container_sas, upload_file)
 
 DATA_PROD_ACCOUNT = "commadata2"

selfdrive/thermald/thermald.py

@@ -312,6 +312,9 @@ def thermald_thread(end_event, hw_queue) -> None:
     # must be at an engageable thermal band to go onroad
     startup_conditions["device_temp_engageable"] = thermal_status < ThermalStatus.red
 
+    # ensure device is fully booted
+    startup_conditions["device_booted"] = startup_conditions.get("device_booted", False) or HARDWARE.booted()
+
     # if the temperature enters the danger zone, go offroad to cool down
     onroad_conditions["device_temp_good"] = thermal_status < ThermalStatus.danger
     extra_text = f"{offroad_comp_temp:.1f}C"

selfdrive/ui/qt/maps/map_eta.cc

@@ -20,7 +20,7 @@ void MapETA::paintEvent(QPaintEvent *event) {
     QPainter p(this);
     p.setRenderHint(QPainter::Antialiasing);
     p.setPen(Qt::NoPen);
-    p.setBrush(QColor(0, 0, 0, 150));
+    p.setBrush(QColor(0, 0, 0, 255));
     QSizeF txt_size = eta_doc.size();
     p.drawRoundedRect((width() - txt_size.width()) / 2 - UI_BORDER_SIZE, 0, txt_size.width() + UI_BORDER_SIZE * 2, height() + 25, 25, 25);
     p.translate((width() - txt_size.width()) / 2, (height() - txt_size.height()) / 2);

selfdrive/ui/qt/network/networking.cc

@@ -48,6 +48,7 @@ Networking::Networking(QWidget* parent, bool show_advanced) : QFrame(parent) {
 
   an = new AdvancedNetworking(this, wifi);
   connect(an, &AdvancedNetworking::backPress, [=]() { main_layout->setCurrentWidget(wifiScreen); });
+  connect(an, &AdvancedNetworking::requestWifiScreen, [=]() { main_layout->setCurrentWidget(wifiScreen); });
   main_layout->addWidget(an);
 
   QPalette pal = palette();
@@ -181,6 +182,25 @@ AdvancedNetworking::AdvancedNetworking(QWidget* parent, WifiManager* wifi): QWid
   });
   list->addItem(meteredToggle);
 
+  // Hidden Network
+  hiddenNetworkButton = new ButtonControl(tr("Hidden Network"), tr("CONNECT"));
+  connect(hiddenNetworkButton, &ButtonControl::clicked, [=]() {
+    QString ssid = InputDialog::getText(tr("Enter SSID"), this, "", false, 1);
+    if (!ssid.isEmpty()) {
+      QString pass = InputDialog::getText(tr("Enter password"), this, tr("for \"%1\"").arg(ssid), true, -1);
+      Network hidden_network;
+      hidden_network.ssid = ssid.toUtf8();
+      if (!pass.isEmpty()) {
+        hidden_network.security_type = SecurityType::WPA;
+        wifi->connect(hidden_network, pass);
+      } else {
+        wifi->connect(hidden_network);
+      }
+      emit requestWifiScreen();
+    }
+  });
+  list->addItem(hiddenNetworkButton);
+
   // Set initial config
   wifi->updateGsmSettings(roamingEnabled, QString::fromStdString(params.get("GsmApn")), metered);
 

selfdrive/ui/qt/network/networking.h

@@ -62,12 +62,14 @@ private:
   ToggleControl* tetheringToggle;
   ToggleControl* roamingToggle;
   ButtonControl* editApnButton;
+  ButtonControl* hiddenNetworkButton;
   ToggleControl* meteredToggle;
   WifiManager* wifi = nullptr;
   Params params;
 
 signals:
   void backPress();
+  void requestWifiScreen();
 
 public slots:
   void toggleTethering(bool enabled);

selfdrive/ui/tests/test_translations.py

@@ -2,30 +2,28 @@
 import json
 import os
 import re
-import shutil
 import unittest
+import shutil
+import tempfile
 import xml.etree.ElementTree as ET
+import string
+import requests
+from parameterized import parameterized_class
 
 from openpilot.selfdrive.ui.update_translations import TRANSLATIONS_DIR, LANGUAGES_FILE, update_translations
 
-TMP_TRANSLATIONS_DIR = os.path.join(TRANSLATIONS_DIR, "tmp")
+with open(LANGUAGES_FILE, "r") as f:
+  translation_files = json.load(f)
+
 UNFINISHED_TRANSLATION_TAG = "<translation type=\"unfinished\""  # non-empty translations can be marked unfinished
 LOCATION_TAG = "<location "
 FORMAT_ARG = re.compile("%[0-9]+")
 
 
+@parameterized_class(("name", "file"), translation_files.items())
 class TestTranslations(unittest.TestCase):
-  @classmethod
+  name: str
-  def setUpClass(cls):
+  file: str
-    with open(LANGUAGES_FILE, "r") as f:
-      cls.translation_files = json.load(f)
-
-    # Set up temp directory
-    shutil.copytree(TRANSLATIONS_DIR, TMP_TRANSLATIONS_DIR, dirs_exist_ok=True)
-
-  @classmethod
-  def tearDownClass(cls):
-    shutil.rmtree(TMP_TRANSLATIONS_DIR, ignore_errors=True)
 
   @staticmethod
   def _read_translation_file(path, file):
@@ -34,39 +32,29 @@ class TestTranslations(unittest.TestCase):
       return f.read()
 
   def test_missing_translation_files(self):
-    for name, file in self.translation_files.items():
+    self.assertTrue(os.path.exists(os.path.join(TRANSLATIONS_DIR, f"{self.file}.ts")),
-      with self.subTest(name=name, file=file):
+                    f"{self.name} has no XML translation file, run selfdrive/ui/update_translations.py")
-        self.assertTrue(os.path.exists(os.path.join(TRANSLATIONS_DIR, f"{file}.ts")),
-                        f"{name} has no XML translation file, run selfdrive/ui/update_translations.py")
 
   def test_translations_updated(self):
-    update_translations(plural_only=["main_en"], translations_dir=TMP_TRANSLATIONS_DIR)
+    with tempfile.TemporaryDirectory() as tmpdir:
-
+      shutil.copytree(TRANSLATIONS_DIR, tmpdir, dirs_exist_ok=True)
-    for name, file in self.translation_files.items():
+      update_translations(translation_files=[self.file], translations_dir=tmpdir)
-      with self.subTest(name=name, file=file):
-        # caught by test_missing_translation_files
-        if not os.path.exists(os.path.join(TRANSLATIONS_DIR, f"{file}.ts")):
-          self.skipTest(f"{name} missing translation file")
 
-        cur_translations = self._read_translation_file(TRANSLATIONS_DIR, file)
+      cur_translations = self._read_translation_file(TRANSLATIONS_DIR, self.file)
-        new_translations = self._read_translation_file(TMP_TRANSLATIONS_DIR, file)
+      new_translations = self._read_translation_file(tmpdir, self.file)
-        self.assertEqual(cur_translations, new_translations,
+      self.assertEqual(cur_translations, new_translations,
-                         f"{file} ({name}) XML translation file out of date. Run selfdrive/ui/update_translations.py to update the translation files")
+                       f"{self.file} ({self.name}) XML translation file out of date. Run selfdrive/ui/update_translations.py to update the translation files")
 
   @unittest.skip("Only test unfinished translations before going to release")
   def test_unfinished_translations(self):
-    for name, file in self.translation_files.items():
+    cur_translations = self._read_translation_file(TRANSLATIONS_DIR, self.file)
-      with self.subTest(name=name, file=file):
+    self.assertTrue(UNFINISHED_TRANSLATION_TAG not in cur_translations,
-        cur_translations = self._read_translation_file(TRANSLATIONS_DIR, file)
+                    f"{self.file} ({self.name}) translation file has unfinished translations. Finish translations or mark them as completed in Qt Linguist")
-        self.assertTrue(UNFINISHED_TRANSLATION_TAG not in cur_translations,
-                        f"{file} ({name}) translation file has unfinished translations. Finish translations or mark them as completed in Qt Linguist")
 
   def test_vanished_translations(self):
-    for name, file in self.translation_files.items():
+    cur_translations = self._read_translation_file(TRANSLATIONS_DIR, self.file)
-      with self.subTest(name=name, file=file):
+    self.assertTrue("<translation type=\"vanished\">" not in cur_translations,
-        cur_translations = self._read_translation_file(TRANSLATIONS_DIR, file)
+                    f"{self.file} ({self.name}) translation file has obsolete translations. Run selfdrive/ui/update_translations.py --vanish to remove them")
-        self.assertTrue("<translation type=\"vanished\">" not in cur_translations,
-                        f"{file} ({name}) translation file has obsolete translations. Run selfdrive/ui/update_translations.py --vanish to remove them")
 
   def test_finished_translations(self):
     """
@@ -78,48 +66,68 @@ class TestTranslations(unittest.TestCase):
       - that translation is not empty
       - that translation format arguments are consistent
     """
-    for name, file in self.translation_files.items():
+    tr_xml = ET.parse(os.path.join(TRANSLATIONS_DIR, f"{self.file}.ts"))
-      with self.subTest(name=name, file=file):
-        tr_xml = ET.parse(os.path.join(TRANSLATIONS_DIR, f"{file}.ts"))
 
-        for context in tr_xml.getroot():
+    for context in tr_xml.getroot():
-          for message in context.iterfind("message"):
+      for message in context.iterfind("message"):
-            translation = message.find("translation")
+        translation = message.find("translation")
-            source_text = message.find("source").text
+        source_text = message.find("source").text
 
-            # Do not test unfinished translations
+        # Do not test unfinished translations
-            if translation.get("type") == "unfinished":
+        if translation.get("type") == "unfinished":
-              continue
+          continue
 
-            if message.get("numerus") == "yes":
+        if message.get("numerus") == "yes":
-              numerusform = [t.text for t in translation.findall("numerusform")]
+          numerusform = [t.text for t in translation.findall("numerusform")]
 
-              for nf in numerusform:
+          for nf in numerusform:
-                self.assertIsNotNone(nf, f"Ensure all plural translation forms are completed: {source_text}")
+            self.assertIsNotNone(nf, f"Ensure all plural translation forms are completed: {source_text}")
-                self.assertIn("%n", nf, "Ensure numerus argument (%n) exists in translation.")
+            self.assertIn("%n", nf, "Ensure numerus argument (%n) exists in translation.")
-                self.assertIsNone(FORMAT_ARG.search(nf), "Plural translations must use %n, not %1, %2, etc.: {}".format(numerusform))
+            self.assertIsNone(FORMAT_ARG.search(nf), "Plural translations must use %n, not %1, %2, etc.: {}".format(numerusform))
 
-            else:
+        else:
-              self.assertIsNotNone(translation.text, f"Ensure translation is completed: {source_text}")
+          self.assertIsNotNone(translation.text, f"Ensure translation is completed: {source_text}")
 
-              source_args = FORMAT_ARG.findall(source_text)
+          source_args = FORMAT_ARG.findall(source_text)
-              translation_args = FORMAT_ARG.findall(translation.text)
+          translation_args = FORMAT_ARG.findall(translation.text)
-              self.assertEqual(sorted(source_args), sorted(translation_args),
+          self.assertEqual(sorted(source_args), sorted(translation_args),
-                               f"Ensure format arguments are consistent: `{source_text}` vs. `{translation.text}`")
+                           f"Ensure format arguments are consistent: `{source_text}` vs. `{translation.text}`")
 
   def test_no_locations(self):
-    for name, file in self.translation_files.items():
+    for line in self._read_translation_file(TRANSLATIONS_DIR, self.file).splitlines():
-      with self.subTest(name=name, file=file):
+      self.assertFalse(line.strip().startswith(LOCATION_TAG),
-        for line in self._read_translation_file(TRANSLATIONS_DIR, file).splitlines():
+                       f"Line contains location tag: {line.strip()}, remove all line numbers.")
-          self.assertFalse(line.strip().startswith(LOCATION_TAG),
-                           f"Line contains location tag: {line.strip()}, remove all line numbers.")
 
   def test_entities_error(self):
-    for name, file in self.translation_files.items():
+    cur_translations = self._read_translation_file(TRANSLATIONS_DIR, self.file)
-      with self.subTest(name=name, file=file):
+    matches = re.findall(r'@(\w+);', cur_translations)
-        cur_translations = self._read_translation_file(TRANSLATIONS_DIR, file)
+    self.assertEqual(len(matches), 0, f"The string(s) {matches} were found with '@' instead of '&'")
-        matches = re.findall(r'@(\w+);', cur_translations)
+
-        self.assertEqual(len(matches), 0, f"The string(s) {matches} were found with '@' instead of '&'")
+  def test_bad_language(self):
+    IGNORED_WORDS = {'pédale'}
+
+    match = re.search(r'_([a-zA-Z]{2,3})', self.file)
+    assert match, f"{self.name} - could not parse language"
+
+    response = requests.get(f"https://raw.githubusercontent.com/LDNOOBW/List-of-Dirty-Naughty-Obscene-and-Otherwise-Bad-Words/master/{match.group(1)}")
+    response.raise_for_status()
+
+    banned_words = {line.strip() for line in response.text.splitlines()}
+
+    for context in ET.parse(os.path.join(TRANSLATIONS_DIR, f"{self.file}.ts")).getroot():
+      for message in context.iterfind("message"):
+        translation = message.find("translation")
+        if translation.get("type") == "unfinished":
+          continue
+
+        translation_text = " ".join([t.text for t in translation.findall("numerusform")]) if message.get("numerus") == "yes" else translation.text
+
+        if not translation_text:
+          continue
+
+        words = set(translation_text.translate(str.maketrans('', '', string.punctuation + '%n')).lower().split())
+        bad_words_found = words & (banned_words - IGNORED_WORDS)
+        assert not bad_words_found, f"Bad language found in {self.name}: '{translation_text}'. Bad word(s): {', '.join(bad_words_found)}"
 
 
 if __name__ == "__main__":

selfdrive/ui/translations/auto_translate.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import os
+import pathlib
+import xml.etree.ElementTree as ET
+from typing import cast
+
+import requests
+
+TRANSLATIONS_DIR = pathlib.Path(__file__).resolve().parent
+TRANSLATIONS_LANGUAGES = TRANSLATIONS_DIR / "languages.json"
+
+OPENAI_MODEL = "gpt-4"
+OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
+OPENAI_PROMPT = "You are a professional translator from English to {language} (ISO 639 language code). " + \
+                "The following sentence or word is in the GUI of a software called openpilot, translate it accordingly."
+
+
+def get_language_files(languages: list[str] | None = None) -> dict[str, pathlib.Path]:
+  files = {}
+
+  with open(TRANSLATIONS_LANGUAGES) as fp:
+    language_dict = json.load(fp)
+
+    for filename in language_dict.values():
+      path = TRANSLATIONS_DIR / f"{filename}.ts"
+      language = path.stem.split("main_")[1]
+
+      if languages is None or language in languages:
+        files[language] = path
+
+  return files
+
+
+def translate_phrase(text: str, language: str) -> str:
+  response = requests.post(
+    "https://api.openai.com/v1/chat/completions",
+    json={
+      "model": OPENAI_MODEL,
+      "messages": [
+        {
+          "role": "system",
+          "content": OPENAI_PROMPT.format(language=language),
+        },
+        {
+          "role": "user",
+          "content": text,
+        },
+      ],
+      "temperature": 0.8,
+      "max_tokens": 1024,
+      "top_p": 1,
+    },
+    headers={
+      "Authorization": f"Bearer {OPENAI_API_KEY}",
+      "Content-Type": "application/json",
+    },
+  )
+
+  if 400 <= response.status_code < 600:
+    raise requests.HTTPError(f'Error {response.status_code}: {response.json()}', response=response)
+
+  data = response.json()
+
+  return cast(str, data["choices"][0]["message"]["content"])
+
+
+def translate_file(path: pathlib.Path, language: str, all_: bool) -> None:
+  tree = ET.parse(path)
+
+  root = tree.getroot()
+
+  for context in root.findall("./context"):
+    name = context.find("name")
+    if name is None:
+      raise ValueError("name not found")
+
+    print(f"Context: {name.text}")
+
+    for message in context.findall("./message"):
+      source = message.find("source")
+      translation = message.find("translation")
+
+      if source is None or translation is None:
+        raise ValueError("source or translation not found")
+
+      if not all_ and translation.attrib.get("type") != "unfinished":
+        continue
+
+      llm_translation = translate_phrase(cast(str, source.text), language)
+
+      print(f"Source: {source.text}\n" +
+            f"Current translation: {translation.text}\n" +
+            f"LLM translation: {llm_translation}")
+
+      translation.text = llm_translation
+
+  with path.open("w", encoding="utf-8") as fp:
+    fp.write('<?xml version="1.0" encoding="utf-8"?>\n' +
+             '<!DOCTYPE TS>\n' +
+             ET.tostring(root, encoding="utf-8").decode())
+
+
+def main():
+  arg_parser = argparse.ArgumentParser("Auto translate")
+
+  group = arg_parser.add_mutually_exclusive_group(required=True)
+  group.add_argument("-a", "--all-files", action="store_true", help="Translate all files")
+  group.add_argument("-f", "--file", nargs="+", help="Translate the selected files. (Example: -f fr de)")
+
+  arg_parser.add_argument("-t", "--all-translations", action="store_true", default=False, help="Translate all sections. (Default: only unfinished)")
+
+  args = arg_parser.parse_args()
+
+  if OPENAI_API_KEY is None:
+    print("OpenAI API key is missing. (Hint: use `export OPENAI_API_KEY=YOUR-KEY` before you run the script).\n" +
+          "If you don't have one go to: https://beta.openai.com/account/api-keys.")
+    exit(1)
+
+  files = get_language_files(None if args.all_files else args.file)
+
+  if args.file:
+    missing_files = set(args.file) - set(files)
+    if len(missing_files):
+      print(f"No language files found: {missing_files}")
+      exit(1)
+
+  print(f"Translation mode: {'all' if args.all_translations else 'only unfinished'}. Files: {list(files)}")
+
+  for lang, path in files.items():
+    print(f"Translate {lang} ({path})")
+    translate_file(path, lang, args.all_translations)
+
+
+if __name__ == "__main__":
+  main()

selfdrive/ui/translations/create_badges.py

@@ -54,8 +54,8 @@ if __name__ == "__main__":
 
     badge_svg.extend([f'<g transform="translate(0, {idx * BADGE_HEIGHT})">', content_svg, "</g>"])
 
-  badge_svg.insert(0, f'<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" \
+  badge_svg.insert(0, '<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" ' +
-                             height="{len(translation_files) * BADGE_HEIGHT}" width="{max_badge_width}">')
+                   f'height="{len(translation_files) * BADGE_HEIGHT}" width="{max_badge_width}">')
   badge_svg.append("</svg>")
 
   with open(os.path.join(BASEDIR, "translation_badge.svg"), "w") as badge_f:

selfdrive/ui/translations/main_ar.ts

@@ -66,6 +66,26 @@
         <source>Prevent large data uploads when on a metered connection</source>
         <translation>منع تحميل البيانات الكبيرة عندما يكون الاتصال محدوداً</translation>
     </message>
+    <message>
+        <source>Hidden Network</source>
+        <translation>شبكة مخفية</translation>
+    </message>
+    <message>
+        <source>CONNECT</source>
+        <translation>الاتصال</translation>
+    </message>
+    <message>
+        <source>Enter SSID</source>
+        <translation>أدخل SSID</translation>
+    </message>
+    <message>
+        <source>Enter password</source>
+        <translation>أدخل كلمة المرور</translation>
+    </message>
+    <message>
+        <source>for &quot;%1&quot;</source>
+        <translation>من أجل &quot;%1&quot;</translation>
+    </message>
 </context>
 <context>
     <name>AnnotatedCameraWidget</name>

selfdrive/ui/translations/main_de.ts

@@ -66,6 +66,26 @@
         <source>Prevent large data uploads when on a metered connection</source>
         <translation>Hochladen großer Dateien über getaktete Verbindungen unterbinden</translation>
     </message>
+    <message>
+        <source>Hidden Network</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <source>CONNECT</source>
+        <translation type="unfinished">CONNECT</translation>
+    </message>
+    <message>
+        <source>Enter SSID</source>
+        <translation type="unfinished">SSID eingeben</translation>
+    </message>
+    <message>
+        <source>Enter password</source>
+        <translation type="unfinished">Passwort eingeben</translation>
+    </message>
+    <message>
+        <source>for &quot;%1&quot;</source>
+        <translation type="unfinished">für &quot;%1&quot;</translation>
+    </message>
 </context>
 <context>
     <name>AnnotatedCameraWidget</name>

selfdrive/ui/translations/main_fr.ts

@@ -66,6 +66,26 @@
         <source>Prevent large data uploads when on a metered connection</source>
         <translation>Éviter les transferts de données importants sur une connexion limitée</translation>
     </message>
+    <message>
+        <source>Hidden Network</source>
+        <translation>Réseau Caché</translation>
+    </message>
+    <message>
+        <source>CONNECT</source>
+        <translation>CONNECTER</translation>
+    </message>
+    <message>
+        <source>Enter SSID</source>
+        <translation>Entrer le SSID</translation>
+    </message>
+    <message>
+        <source>Enter password</source>
+        <translation>Entrer le mot de passe</translation>
+    </message>
+    <message>
+        <source>for &quot;%1&quot;</source>
+        <translation>pour &quot;%1&quot;</translation>
+    </message>
 </context>
 <context>
     <name>AnnotatedCameraWidget</name>

selfdrive/ui/translations/main_ja.ts

@@ -66,6 +66,26 @@
         <source>Prevent large data uploads when on a metered connection</source>
         <translation>大量のデータのアップロードを防止します。</translation>
     </message>
+    <message>
+        <source>Hidden Network</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <source>CONNECT</source>
+        <translation type="unfinished">接続</translation>
+    </message>
+    <message>
+        <source>Enter SSID</source>
+        <translation type="unfinished">SSID を入力</translation>
+    </message>
+    <message>
+        <source>Enter password</source>
+        <translation type="unfinished">パスワードを入力</translation>
+    </message>
+    <message>
+        <source>for &quot;%1&quot;</source>
+        <translation type="unfinished">ﾈｯﾄﾜｰｸ名：%1</translation>
+    </message>
 </context>
 <context>
     <name>AnnotatedCameraWidget</name>

selfdrive/ui/translations/main_ko.ts

@@ -66,6 +66,26 @@
         <source>Prevent large data uploads when on a metered connection</source>
         <translation>데이터 요금제 연결 시 대용량 데이터 업로드를 방지합니다</translation>
     </message>
+    <message>
+        <source>Hidden Network</source>
+        <translation>숨겨진 네트워크</translation>
+    </message>
+    <message>
+        <source>CONNECT</source>
+        <translation>연결됨</translation>
+    </message>
+    <message>
+        <source>Enter SSID</source>
+        <translation>SSID 입력</translation>
+    </message>
+    <message>
+        <source>Enter password</source>
+        <translation>비밀번호를 입력하세요</translation>
+    </message>
+    <message>
+        <source>for &quot;%1&quot;</source>
+        <translation>&quot;%1&quot;에 접속하려면 비밀번호가 필요합니다</translation>
+    </message>
 </context>
 <context>
     <name>AnnotatedCameraWidget</name>

selfdrive/ui/translations/main_pt-BR.ts

@@ -66,6 +66,26 @@
         <source>Prevent large data uploads when on a metered connection</source>
         <translation>Evite grandes uploads de dados quando estiver em uma conexão limitada</translation>
     </message>
+    <message>
+        <source>Hidden Network</source>
+        <translation>Rede Oculta</translation>
+    </message>
+    <message>
+        <source>CONNECT</source>
+        <translation>CONECTE</translation>
+    </message>
+    <message>
+        <source>Enter SSID</source>
+        <translation>Digite o SSID</translation>
+    </message>
+    <message>
+        <source>Enter password</source>
+        <translation>Insira a senha</translation>
+    </message>
+    <message>
+        <source>for &quot;%1&quot;</source>
+        <translation>para &quot;%1&quot;</translation>
+    </message>
 </context>
 <context>
     <name>AnnotatedCameraWidget</name>

selfdrive/ui/translations/main_th.ts

@@ -66,6 +66,26 @@
         <source>Prevent large data uploads when on a metered connection</source>
         <translation>ปิดการอัพโหลดข้อมูลขนาดใหญ่เมื่อเชื่อมต่อผ่านเซลลูล่าร์</translation>
     </message>
+    <message>
+        <source>Hidden Network</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <source>CONNECT</source>
+        <translation type="unfinished">เชื่อมต่อ</translation>
+    </message>
+    <message>
+        <source>Enter SSID</source>
+        <translation type="unfinished">ป้อนค่า SSID</translation>
+    </message>
+    <message>
+        <source>Enter password</source>
+        <translation type="unfinished">ใส่รหัสผ่าน</translation>
+    </message>
+    <message>
+        <source>for &quot;%1&quot;</source>
+        <translation type="unfinished">สำหรับ &quot;%1&quot;</translation>
+    </message>
 </context>
 <context>
     <name>AnnotatedCameraWidget</name>

selfdrive/ui/translations/main_tr.ts

@@ -66,6 +66,26 @@
         <source>Prevent large data uploads when on a metered connection</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <source>Hidden Network</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <source>CONNECT</source>
+        <translation type="unfinished">BAĞLANTI</translation>
+    </message>
+    <message>
+        <source>Enter SSID</source>
+        <translation type="unfinished">APN Gir</translation>
+    </message>
+    <message>
+        <source>Enter password</source>
+        <translation type="unfinished">Parolayı girin</translation>
+    </message>
+    <message>
+        <source>for &quot;%1&quot;</source>
+        <translation type="unfinished">için &quot;%1&quot;</translation>
+    </message>
 </context>
 <context>
     <name>AnnotatedCameraWidget</name>

selfdrive/ui/translations/main_zh-CHS.ts

@@ -66,6 +66,26 @@
         <source>Prevent large data uploads when on a metered connection</source>
         <translation>当使用按流量计费的连接时，避免上传大流量数据</translation>
     </message>
+    <message>
+        <source>Hidden Network</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <source>CONNECT</source>
+        <translation type="unfinished">CONNECT</translation>
+    </message>
+    <message>
+        <source>Enter SSID</source>
+        <translation type="unfinished">输入SSID</translation>
+    </message>
+    <message>
+        <source>Enter password</source>
+        <translation type="unfinished">输入密码</translation>
+    </message>
+    <message>
+        <source>for &quot;%1&quot;</source>
+        <translation type="unfinished">网络名称：&quot;%1&quot;</translation>
+    </message>
 </context>
 <context>
     <name>AnnotatedCameraWidget</name>

selfdrive/ui/translations/main_zh-CHT.ts

@@ -66,6 +66,26 @@
         <source>Prevent large data uploads when on a metered connection</source>
         <translation>防止使用行動網路上傳大量的數據</translation>
     </message>
+    <message>
+        <source>Hidden Network</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <source>CONNECT</source>
+        <translation type="unfinished">雲端服務</translation>
+    </message>
+    <message>
+        <source>Enter SSID</source>
+        <translation type="unfinished">輸入 SSID</translation>
+    </message>
+    <message>
+        <source>Enter password</source>
+        <translation type="unfinished">輸入密碼</translation>
+    </message>
+    <message>
+        <source>for &quot;%1&quot;</source>
+        <translation type="unfinished">給 &quot;%1&quot;</translation>
+    </message>
 </context>
 <context>
     <name>AnnotatedCameraWidget</name>

selfdrive/ui/update_translations.py

@@ -9,6 +9,7 @@ UI_DIR = os.path.join(BASEDIR, "selfdrive", "ui")
 TRANSLATIONS_DIR = os.path.join(UI_DIR, "translations")
 LANGUAGES_FILE = os.path.join(TRANSLATIONS_DIR, "languages.json")
 TRANSLATIONS_INCLUDE_FILE = os.path.join(TRANSLATIONS_DIR, "alerts_generated.h")
+PLURAL_ONLY = ["main_en"]  # base language, only create entries for strings with plural forms
 
 
 def generate_translations_include():
@@ -22,21 +23,20 @@ def generate_translations_include():
   with open(TRANSLATIONS_INCLUDE_FILE, "w") as f:
     f.write(content)
 
-def update_translations(vanish=False, plural_only=None, translations_dir=TRANSLATIONS_DIR):
-  generate_translations_include()
 
-  if plural_only is None:
+def update_translations(vanish: bool = False, translation_files: None | list[str] = None, translations_dir: str = TRANSLATIONS_DIR):
-    plural_only = []
+  generate_translations_include()
 
-  with open(LANGUAGES_FILE, "r") as f:
+  if translation_files is None:
-    translation_files = json.load(f)
+    with open(LANGUAGES_FILE, "r") as f:
+      translation_files = json.load(f).values()
 
-  for file in translation_files.values():
+  for file in translation_files:
     tr_file = os.path.join(translations_dir, f"{file}.ts")
     args = f"lupdate -locations none -recursive {UI_DIR} -ts {tr_file} -I {BASEDIR}"
     if vanish:
       args += " -no-obsolete"
-    if file in plural_only:
+    if file in PLURAL_ONLY:
       args += " -pluralonly"
     ret = os.system(args)
     assert ret == 0
@@ -46,8 +46,6 @@ if __name__ == "__main__":
   parser = argparse.ArgumentParser(description="Update translation files for UI",
                                    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
   parser.add_argument("--vanish", action="store_true", help="Remove translations with source text no longer found")
-  parser.add_argument("--plural-only", type=str, nargs="*", default=["main_en"],
-                      help="Translation codes to only create plural translations for (ie. the base language)")
   args = parser.parse_args()
 
-  update_translations(args.vanish, args.plural_only)
+  update_translations(args.vanish)