rebase

.devcontainer/devcontainer.json

@@ -5,7 +5,7 @@
   },
   "postCreateCommand": ".devcontainer/container_post_create.sh",
   "postStartCommand": ".devcontainer/container_post_start.sh",
-  "initializeCommand": ".devcontainer/host_setup.sh",
+  "initializeCommand": [".devcontainer/host_setup"],
   "privileged": true,
   "containerEnv": {
     "DISPLAY": "${localEnv:DISPLAY}",

.devcontainer/host_setup.cmd

@@ -0,0 +1,10 @@
+:: pull base image
+IF NOT DEFINED USE_LOCAL_IMAGE ^
+echo "Updating openpilot_base image if needed..." && ^
+docker pull ghcr.io/commaai/openpilot-base:latest
+
+:: setup .host dir
+mkdir .devcontainer\.host
+
+:: setup host env file
+echo "" > .devcontainer\.host\.env

.dockerignore

@@ -28,10 +28,8 @@ chffr/app2
 chffr/backend/env
 selfdrive/nav
 selfdrive/baseui
-chffr/lib/vidindex/vidindex
 selfdrive/test/simulator2
 **/cache_data
-xx/chffr/lib/vidindex/vidindex
 xx/plus
 xx/community
 xx/projects

.github/labeler.yaml

@@ -1,8 +1,6 @@
 CI / testing:
   - all:
-    - changed-files: ['.github/**']
+    - changed-files: ['.github/**', '**/test_*', 'Jenkinsfile']
-  - all:
-    - changed-files: ['**/test_*']
 
 car: 
   - all:
@@ -59,3 +57,9 @@ multilanguage:
   - all:
     - changed-files: ['selfdrive/ui/translations/**']
 
+research:
+  - all:
+    - changed-files: [
+        'selfdrive/modeld/models/**',
+        'selfdrive/test/process_replay/model_replay_ref_commit',
+      ]

.github/workflows/selfdrive_tests.yaml

@@ -228,6 +228,33 @@ jobs:
     - name: "Upload coverage to Codecov"
       uses: codecov/codecov-action@v3
 
+  regen:
+    name: regen
+    runs-on: 'ubuntu-20.04'
+    steps: 
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Cache test routes
+      id: dependency-cache
+      uses: actions/cache@v3
+      with:
+        path: .ci_cache/comma_download_cache
+        key: regen-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/test_regen.py') }}
+    - name: Build base Docker image
+      run: eval "$BUILD"
+    - name: Build Docker image
+      run: eval "$BUILD_CL"
+    - name: Build openpilot
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+    - name: Run regen
+      timeout-minutes: 30
+      run: |
+        ${{ env.RUN_CL }} "ONNXCPU=1 $PYTEST -n auto --dist=loadscope selfdrive/test/process_replay/test_regen.py && \
+                           chmod -R 777 /tmp/comma_download_cache"
+
   test_modeld:
     name: model tests
     runs-on: ubuntu-20.04

.github/workflows/tools_tests.yaml

@@ -106,3 +106,4 @@ jobs:
         devcontainer exec --workspace-folder . scons -j$(nproc)
         devcontainer exec --workspace-folder . pip install pip-install-test
         devcontainer exec --workspace-folder . touch /home/batman/.comma/auth.json
+        devcontainer exec --workspace-folder . sudo touch /root/test.txt

.pre-commit-config.yaml

@@ -23,13 +23,13 @@ repos:
         - --maxkb=500
         - --enforce-all
 -   repo: https://github.com/codespell-project/codespell
-    rev: v2.2.5
+    rev: v2.2.6
     hooks:
     -   id: codespell
         exclude: '^(third_party/)|(body/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(opendbc/)|(laika_repo/)|(rednose_repo/)|(selfdrive/ui/translations/.*.ts)|(poetry.lock)'
         args:
         # if you've got a short variable name that's getting flagged, add it here
-        - -L bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie
+        - -L bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,preints
         - --builtins clear,rare,informal,usage,code,names,en-GB_to_en-US
 -   repo: local
     hooks:
@@ -41,7 +41,7 @@ repos:
         args: ['--explicit-package-bases']
         exclude: '^(third_party/)|(cereal/)|(opendbc/)|(panda/)|(laika/)|(laika_repo/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(xx/)'
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.0.291
+    rev: v0.0.292
     hooks:
     -   id: ruff
         exclude: '^(third_party/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(laika_repo/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)'

Dockerfile.openpilot_base

@@ -25,6 +25,8 @@ RUN cd /tmp && \
 ARG USER=batman
 ARG USER_UID=1000
 RUN useradd -m -s /bin/bash -u $USER_UID $USER
+RUN usermod -aG sudo $USER
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 USER $USER
 
 ENV POETRY_VIRTUALENVS_CREATE=false

Jenkinsfile

@@ -81,10 +81,14 @@ def pcStage(String stageName, Closure body) {
 
     checkout scm
 
-    def dockerArgs = '--user=root -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/scons_cache:/tmp/scons_cache';
+    def dockerArgs = '--user=batman -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/scons_cache:/tmp/scons_cache';
     docker.build("openpilot-base:build-${env.GIT_COMMIT}", "-f Dockerfile.openpilot_base .").inside(dockerArgs) {
       timeout(time: 20, unit: 'MINUTES') {
         try {
+          // TODO: remove these after all jenkins jobs are running as batman (merged with master)
+          sh "sudo chown -R batman:batman /tmp/scons_cache"
+          sh "sudo chown -R batman:batman /tmp/comma_download_cache"
+
           sh "git config --global --add safe.directory '*'"
           sh "git submodule update --init --recursive"
           sh "git lfs pull"

cereal

@@ -1 +1 @@
-Subproject commit 68fe5d1f9d255aa3f9577b50fc1e1fc138a74fd3
+Subproject commit bcdbfcfdeb4bf59eae484e9b2c726c8b41c87350

codecov.yml

@@ -6,3 +6,5 @@ coverage:
         informational: true
     patch: off
 
+ignore:
+  - "**/test_*.py"

common/params.cc

@@ -88,7 +88,6 @@ private:
 std::unordered_map<std::string, uint32_t> keys = {
     {"AccessToken", CLEAR_ON_MANAGER_START | DONT_LOG},
     {"ApiCache_Device", PERSISTENT},
-    {"ApiCache_DriveStats", PERSISTENT},
     {"ApiCache_NavDestinations", PERSISTENT},
     {"AssistNowToken", PERSISTENT},
     {"AthenadPid", PERSISTENT},
@@ -181,7 +180,6 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"Offroad_UpdateFailed", CLEAR_ON_MANAGER_START},
     {"OpenpilotEnabledToggle", PERSISTENT},
     {"PandaHeartbeatLost", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
-    {"PandaLogState", PERSISTENT},
     {"PandaSignatures", CLEAR_ON_MANAGER_START},
     {"Passive", PERSISTENT},
     {"PrimeType", PERSISTENT},

common/transformations/README.md

@@ -11,7 +11,7 @@ by generating a rotation matrix and multiplying.
 | :-------------: |:-------------:| :-----:| :----: |
 | Geodetic | [Latitude, Longitude, Altitude] | geodetic coordinates | Sometimes used as [lon, lat, alt], avoid this frame. |
 | ECEF | [x, y, z] | meters | We use **ITRF14 (IGS14)**, NOT NAD83. <br> This is the global Mesh3D frame. |
-| NED | [North, East, Down] | meters | Relative to earth's surface, useful for vizualizing. |
+| NED | [North, East, Down] | meters | Relative to earth's surface, useful for visualizing. |
 | Device | [Forward, Right, Down] | meters | This is the Mesh3D local frame. <br> Relative to camera, **not imu.** <br> ![img](http://upload.wikimedia.org/wikipedia/commons/thumb/2/2f/RPY_angles_of_airplanes.png/440px-RPY_angles_of_airplanes.png)|
 | Calibrated | [Forward, Right, Down] | meters | This is the frame the model outputs are in. <br> More details below. <br>|
 | Car | [Forward, Right, Down] | meters | This is useful for estimating position of points on the road. <br> More details below. <br>|

conftest.py

@@ -1,10 +1,28 @@
+import os
 import pytest
 
 from openpilot.common.prefix import OpenpilotPrefix
 
 
 @pytest.fixture(scope="function", autouse=True)
-def global_setup_and_teardown():
+def openpilot_function_fixture():
+  starting_env = dict(os.environ)
+
   # setup a clean environment for each test
   with OpenpilotPrefix():
     yield
+
+  os.environ.clear()
+  os.environ.update(starting_env)
+
+
+# If you use setUpClass, the environment variables won't be cleared properly,
+# so we need to hook both the function and class pytest fixtures
+@pytest.fixture(scope="class", autouse=True)
+def openpilot_class_fixture():
+  starting_env = dict(os.environ)
+
+  yield
+
+  os.environ.clear()
+  os.environ.update(starting_env)

docs/CARS.md

@@ -136,7 +136,7 @@ A supported vehicle is one that just works when you install a comma device. All
 |Kia|Optima 2019-20|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai G connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Optima 2019-20">Buy Here</a></sub></details>||
 |Kia|Optima Hybrid 2019|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai H connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Optima Hybrid 2019">Buy Here</a></sub></details>||
 |Kia|Seltos 2021|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai A connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Seltos 2021">Buy Here</a></sub></details>||
-|Kia|Sorento 2018|Advanced Smart Cruise Control|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai C connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Sorento 2018">Buy Here</a></sub></details>|<a href="https://www.youtube.com/watch?v=Fkh3s6WHJz8" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
+|Kia|Sorento 2018|Advanced Smart Cruise Control & LKAS|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai E connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Sorento 2018">Buy Here</a></sub></details>|<a href="https://www.youtube.com/watch?v=Fkh3s6WHJz8" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Kia|Sorento 2019|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai E connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Sorento 2019">Buy Here</a></sub></details>|<a href="https://www.youtube.com/watch?v=Fkh3s6WHJz8" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Kia|Sorento 2021-23[<sup>6</sup>](#footnotes)|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai K connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Sorento 2021-23">Buy Here</a></sub></details>||
 |Kia|Sorento Hybrid 2023[<sup>6</sup>](#footnotes)|All|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai A connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Sorento Hybrid 2023">Buy Here</a></sub></details>||
@@ -148,7 +148,7 @@ A supported vehicle is one that just works when you install a comma device. All
 |Kia|Telluride 2020-22|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai H connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Kia&model=Telluride 2020-22">Buy Here</a></sub></details>||
 |Lexus|CT Hybrid 2017-18|Lexus Safety System+|openpilot available[<sup>2</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=CT Hybrid 2017-18">Buy Here</a></sub></details>||
 |Lexus|ES 2017-18|All|openpilot available[<sup>2</sup>](#footnotes)|19 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=ES 2017-18">Buy Here</a></sub></details>||
-|Lexus|ES 2019-22|All|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=ES 2019-22">Buy Here</a></sub></details>||
+|Lexus|ES 2019-24|All|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=ES 2019-24">Buy Here</a></sub></details>||
 |Lexus|ES Hybrid 2017-18|All|openpilot available[<sup>2</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=ES Hybrid 2017-18">Buy Here</a></sub></details>||
 |Lexus|ES Hybrid 2019-23|All|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=ES Hybrid 2019-23">Buy Here</a></sub></details>|<a href="https://youtu.be/BZ29osRVJeg?t=12" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Lexus|IS 2017-19|All|Stock|19 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=IS 2017-19">Buy Here</a></sub></details>||
@@ -243,7 +243,7 @@ A supported vehicle is one that just works when you install a comma device. All
 |Volkswagen|Arteon R 2020-23|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 J533 connector<br>- 1 USB-C coupler<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Arteon R 2020-23">Buy Here</a></sub></details>|<a href="https://youtu.be/FAomFKPFlDA" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Volkswagen|Atlas 2018-23|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 J533 connector<br>- 1 USB-C coupler<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Atlas 2018-23">Buy Here</a></sub></details>||
 |Volkswagen|Atlas Cross Sport 2021-22|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 J533 connector<br>- 1 USB-C coupler<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Atlas Cross Sport 2021-22">Buy Here</a></sub></details>||
-|Volkswagen|California 2021|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|31 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 J533 connector<br>- 1 USB-C coupler<br>- 1 angled mount (8 degrees)<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=California 2021">Buy Here</a></sub></details>||
+|Volkswagen|California 2021-23|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|31 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 J533 connector<br>- 1 USB-C coupler<br>- 1 angled mount (8 degrees)<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=California 2021-23">Buy Here</a></sub></details>||
 |Volkswagen|Caravelle 2020|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|31 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 J533 connector<br>- 1 USB-C coupler<br>- 1 angled mount (8 degrees)<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Caravelle 2020">Buy Here</a></sub></details>||
 |Volkswagen|CC 2018-22|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 J533 connector<br>- 1 USB-C coupler<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=CC 2018-22">Buy Here</a></sub></details>|<a href="https://youtu.be/FAomFKPFlDA" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Volkswagen|Crafter 2017-23|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|31 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 J533 connector<br>- 1 USB-C coupler<br>- 1 angled mount (8 degrees)<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Volkswagen&model=Crafter 2017-23">Buy Here</a></sub></details>|<a href="https://youtu.be/4100gLeabmo" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|

panda

@@ -1 +1 @@
-Subproject commit 62db60595bb5129cc2f295fdc42f2cda473777fd
+Subproject commit 6bf6ba773ea4174c702ac9c4af4e8a6fd3cef655

pyproject.toml

@@ -90,7 +90,6 @@ numpy = "*"
 onnx = ">=1.14.0"
 onnxruntime = { version = ">=1.15.1", platform = "linux", markers = "platform_machine == 'aarch64'" }
 onnxruntime-gpu = { version = ">=1.15.1", platform = "linux", markers = "platform_machine == 'x86_64'" }
-pillow = "*"
 psutil = "*"
 pyaudio = "*"
 pycapnp = "*"
@@ -109,11 +108,9 @@ setproctitle = "*"
 smbus2 = "*"
 sounddevice = "*"
 spidev = { version = "*", platform = "linux" }
-spidev2 = { version = "*", platform = "linux" }
 sympy = "*"
 timezonefinder = "*"
 tqdm = "*"
-urllib3 = "*"
 websocket_client = "*"
 polyline = "*"
 sconscontrib = {git = "https://github.com/SCons/scons-contrib.git"}
@@ -124,7 +121,6 @@ av = "*"
 azure-identity = "*"
 azure-storage-blob = "*"
 breathe = "*"
-carla = { url = "https://github.com/commaai/carla/releases/download/3.11.4/carla-0.9.14-cp311-cp311-linux_x86_64.whl", platform = "linux", markers = "platform_machine == 'x86_64'" }
 coverage = "*"
 dictdiffer = "*"
 ft4222 = "*"
@@ -152,6 +148,7 @@ pytest-subtests = "*"
 pytest-xdist = "*"
 pytest-timeout = "*"
 pytest-timeouts = "*"
+pytest-random-order = "*"
 ruff = "*"
 scipy = "*"
 sphinx = "*"
@@ -167,6 +164,11 @@ types-requests = "*"
 types-tabulate = "*"
 pyqt5 = { version = "*", markers = "platform_machine == 'x86_64'" } # no aarch64 wheels for macOS/linux
 
+[tool.poetry.group.carla]
+optional = true
+
+[tool.poetry.group.carla.dependencies]
+carla = { url = "https://github.com/commaai/carla/releases/download/3.11.4/carla-0.9.14-cp311-cp311-linux_x86_64.whl", platform = "linux", markers = "platform_machine == 'x86_64'" }
 
 [build-system]
 requires = ["poetry-core"]

release/files_common

@@ -71,7 +71,6 @@ selfdrive/__init__.py
 selfdrive/sentry.py
 selfdrive/tombstoned.py
 selfdrive/updated.py
-selfdrive/rtshield.py
 selfdrive/statsd.py
 
 system/logmessaged.py
@@ -201,8 +200,6 @@ selfdrive/controls/lib/longitudinal_mpc_lib/.gitignore
 selfdrive/controls/lib/lateral_mpc_lib/*
 selfdrive/controls/lib/longitudinal_mpc_lib/*
 
-selfdrive/hardware
-
 system/__init__.py
 
 system/hardware/__init__.py

selfdrive/athena/athenad.py

@@ -40,7 +40,7 @@ from openpilot.system.loggerd.xattr_cache import getxattr, setxattr
 from openpilot.selfdrive.statsd import STATS_DIR
 from openpilot.system.swaglog import cloudlog
 from openpilot.system.version import get_commit, get_origin, get_short_branch, get_version
-from openpilot.selfdrive.hardware.hw import Paths
+from openpilot.system.hardware.hw import Paths
 
 
 # TODO: use socket constant when mypy recognizes this as a valid attribute

selfdrive/athena/tests/test_athenad.py

@@ -21,7 +21,7 @@ from openpilot.selfdrive.athena import athenad
 from openpilot.selfdrive.athena.athenad import MAX_RETRY_COUNT, dispatcher
 from openpilot.selfdrive.athena.tests.helpers import MockWebsocket, MockParams, MockApi, EchoSocket, with_http_server
 from cereal import messaging
-from openpilot.selfdrive.hardware.hw import Paths
+from openpilot.system.hardware.hw import Paths
 
 
 class TestAthenadMethods(unittest.TestCase):

selfdrive/boardd/pandad.py

@@ -3,7 +3,6 @@
 import os
 import usb1
 import time
-import json
 import subprocess
 from typing import List, NoReturn
 from functools import cmp_to_key
@@ -24,51 +23,6 @@ def get_expected_signature(panda: Panda) -> bytes:
     cloudlog.exception("Error computing expected signature")
     return b""
 
-def read_panda_logs(panda: Panda) -> None:
-  """
-    Forward panda logs to the cloud
-  """
-
-  params = Params()
-  serial = panda.get_usb_serial()
-
-  log_state = {}
-  try:
-    ls = params.get("PandaLogState")
-    if ls is not None:
-      l = json.loads(ls)
-      for k, v in l.items():
-        if isinstance(k, str) and isinstance(v, int):
-          log_state[k] = v
-  except (TypeError, json.JSONDecodeError):
-    cloudlog.exception("failed to parse PandaLogState")
-
-  try:
-    if serial in log_state:
-      logs = panda.get_logs(last_id=log_state[serial])
-    else:
-      logs = panda.get_logs(get_all=True)
-
-    # truncate logs to 100 entries if needed
-    MAX_LOGS = 100
-    if len(logs) > MAX_LOGS:
-      cloudlog.warning(f"Panda {serial} has {len(logs)} logs, truncating to {MAX_LOGS}")
-      logs = logs[-MAX_LOGS:]
-
-    # update log state
-    if len(logs) > 0:
-      log_state[serial] = logs[-1]["id"]
-
-    for log in logs:
-      if log['timestamp'] is not None:
-        log['timestamp'] = log['timestamp'].isoformat()
-      cloudlog.event("panda_log", **log, serial=serial)
-
-    params.put("PandaLogState", json.dumps(log_state))
-  except Exception:
-    cloudlog.exception(f"Error getting logs for panda {serial}")
-
-
 def flash_panda(panda_serial: str) -> Panda:
   try:
     panda = Panda(panda_serial)
@@ -191,8 +145,6 @@ def main() -> NoReturn:
           params.put_bool("PandaHeartbeatLost", True)
           cloudlog.event("heartbeat lost", deviceState=health, serial=panda.get_usb_serial())
 
-        read_panda_logs(panda)
-
         if first_run:
           if panda.is_internal():
             # update time from RTC

selfdrive/boardd/tests/test_pandad.py

@@ -6,11 +6,9 @@ import unittest
 import cereal.messaging as messaging
 from cereal import log
 from openpilot.common.gpio import gpio_set, gpio_init
-from openpilot.common.params import Params
 from panda import Panda, PandaDFU, PandaProtocolMismatch
-from openpilot.selfdrive.test.helpers import phone_only
 from openpilot.selfdrive.manager.process_config import managed_processes
-from openpilot.system.hardware import HARDWARE
+from openpilot.system.hardware import HARDWARE, PC
 from openpilot.system.hardware.tici.pins import GPIO
 
 HERE = os.path.dirname(os.path.realpath(__file__))
@@ -19,8 +17,11 @@ HERE = os.path.dirname(os.path.realpath(__file__))
 class TestPandad(unittest.TestCase):
 
   def setUp(self):
-    self.params = Params()
+    if PC:
-    self.start_log_state = self.params.get("PandaLogState")
+      raise unittest.SkipTest("needs a panda")
+    # ensure panda is up
+    if len(Panda.list()) == 0:
+      self._run_test(60)
 
   def tearDown(self):
     managed_processes['pandad'].stop()
@@ -39,10 +40,6 @@ class TestPandad(unittest.TestCase):
     if sm['peripheralState'].pandaType == log.PandaState.PandaType.unknown:
       raise Exception("boardd failed to start")
 
-    # simple check that we did something with the panda logs
-    cur_log_state = self.params.get("PandaLogState")
-    assert cur_log_state != self.start_log_state
-
   def _go_to_dfu(self):
     HARDWARE.recover_internal_panda()
     assert Panda.wait_for_dfu(None, 10)
@@ -71,19 +68,16 @@ class TestPandad(unittest.TestCase):
 
     self._run_test(45)
 
-  @phone_only
   def test_in_dfu(self):
     HARDWARE.recover_internal_panda()
     self._run_test(60)
 
-  @phone_only
   def test_in_bootstub(self):
     with Panda() as p:
       p.reset(enter_bootstub=True)
       assert p.bootstub
     self._run_test()
 
-  @phone_only
   def test_internal_panda_reset(self):
     gpio_init(GPIO.STM_RST_N, True)
     gpio_set(GPIO.STM_RST_N, 1)
@@ -93,28 +87,23 @@ class TestPandad(unittest.TestCase):
 
     assert any(Panda(s).is_internal() for s in Panda.list())
 
-  @phone_only
   def test_best_case_startup_time(self):
     # run once so we're setup
-    self._run_test()
+    self._run_test(60)
 
     # should be fast this time
     self._run_test(8)
 
-  @phone_only
   def test_protocol_version_check(self):
     if HARDWARE.get_device_type() == 'tici':
-      self.skipTest("")
+      raise unittest.SkipTest("SPI test")
-
     # flash old fw
     fn = os.path.join(HERE, "bootstub.panda_h7_spiv0.bin")
     self._flash_bootstub_and_test(fn, expect_mismatch=True)
 
-  @phone_only
   def test_release_to_devel_bootstub(self):
     self._flash_bootstub_and_test(None)
 
-  @phone_only
   def test_recover_from_bad_bootstub(self):
     self._go_to_dfu()
     with PandaDFU(None) as pd:

selfdrive/car/hyundai/radar_interface.py

@@ -8,6 +8,7 @@ from openpilot.selfdrive.car.hyundai.values import DBC
 RADAR_START_ADDR = 0x500
 RADAR_MSG_COUNT = 32
 
+# POC for parsing corner radars: https://github.com/commaai/openpilot/pull/24221/
 
 def get_radar_can_parser(CP):
   if DBC[CP.carFingerprint]['radar'] is None:

selfdrive/car/hyundai/tests/test_hyundai.py

@@ -62,7 +62,7 @@ class TestHyundaiFingerprint(unittest.TestCase):
     # Asserts no ECUs known to be shared across platforms exist in the database.
     # Tucson having Santa Cruz camera and EPS for example
     for car_model, ecus in FW_VERSIONS.items():
-      with self.subTest(car_model=car_model):
+      with self.subTest(car_model=car_model.value):
         if car_model == CAR.SANTA_CRUZ_1ST_GEN:
           raise unittest.SkipTest("Skip checking Santa Cruz for its parts")
 
@@ -79,7 +79,7 @@ class TestHyundaiFingerprint(unittest.TestCase):
     """
     expected_fw_prefix = HYUNDAI_VERSION_REQUEST_LONG[1:]
     for car_model, ecus in FW_VERSIONS.items():
-      with self.subTest(car_model=car_model):
+      with self.subTest(car_model=car_model.value):
         for ecu, fws in ecus.items():
           # TODO: enable for Ecu.fwdRadar, Ecu.abs, Ecu.eps, Ecu.transmission
           if ecu[0] in (Ecu.fwdCamera,):
@@ -103,7 +103,7 @@ class TestHyundaiFingerprint(unittest.TestCase):
 
     # Asserts ECU keys essential for fuzzy fingerprinting are available on all platforms
     for car_model, ecus in FW_VERSIONS.items():
-      with self.subTest(car_model=car_model):
+      with self.subTest(car_model=car_model.value):
         for platform_code_ecu in PLATFORM_CODE_ECUS:
           if platform_code_ecu in (Ecu.fwdRadar, Ecu.eps) and car_model == CAR.HYUNDAI_GENESIS:
             continue
@@ -118,7 +118,7 @@ class TestHyundaiFingerprint(unittest.TestCase):
     # - expected parsing of ECU FW dates
 
     for car_model, ecus in FW_VERSIONS.items():
-      with self.subTest(car_model=car_model):
+      with self.subTest(car_model=car_model.value):
         for ecu, fws in ecus.items():
           if ecu[0] not in PLATFORM_CODE_ECUS:
             continue

selfdrive/car/hyundai/values.py

@@ -252,8 +252,8 @@ CAR_INFO: Dict[str, Optional[Union[HyundaiCarInfo, List[HyundaiCarInfo]]]] = {
   CAR.KIA_SELTOS: HyundaiCarInfo("Kia Seltos 2021", car_parts=CarParts.common([CarHarness.hyundai_a])),
   CAR.KIA_SPORTAGE_5TH_GEN: HyundaiCarInfo("Kia Sportage 2023", car_parts=CarParts.common([CarHarness.hyundai_n])),
   CAR.KIA_SORENTO: [
-    HyundaiCarInfo("Kia Sorento 2018", "Advanced Smart Cruise Control", video_link="https://www.youtube.com/watch?v=Fkh3s6WHJz8",
+    HyundaiCarInfo("Kia Sorento 2018", "Advanced Smart Cruise Control & LKAS", video_link="https://www.youtube.com/watch?v=Fkh3s6WHJz8",
-                   car_parts=CarParts.common([CarHarness.hyundai_c])),
+                   car_parts=CarParts.common([CarHarness.hyundai_e])),
     HyundaiCarInfo("Kia Sorento 2019", video_link="https://www.youtube.com/watch?v=Fkh3s6WHJz8", car_parts=CarParts.common([CarHarness.hyundai_e])),
   ],
   CAR.KIA_SORENTO_4TH_GEN: HyundaiCarInfo("Kia Sorento 2021-23", car_parts=CarParts.common([CarHarness.hyundai_k])),
@@ -400,7 +400,7 @@ def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]:
   # to distinguish between hybrid and ICE. All EVs so far are either exclusively
   # electric or specify electric in the platform code.
   # TODO: whitelist platforms that we've seen hybrid and ICE versions of that have these specifiers
-  fuzzy_platform_blacklist = {str(car) for car in set(CANFD_CAR - EV_CAR)}
+  fuzzy_platform_blacklist = {str(c) for c in set(CANFD_CAR - EV_CAR)}
   candidates: Set[str] = set()
 
   for candidate, fws in FW_VERSIONS.items():

selfdrive/car/subaru/values.py

@@ -114,7 +114,7 @@ CAR_INFO: Dict[str, Union[SubaruCarInfo, List[SubaruCarInfo]]] = {
     SubaruCarInfo("Subaru XV 2020-21"),
   ],
   # TODO: is there an XV and Impreza too?
-  CAR.CROSSTREK_HYBRID: SubaruCarInfo("Subaru Crosstrek Hybrid 2020"),
+  CAR.CROSSTREK_HYBRID: SubaruCarInfo("Subaru Crosstrek Hybrid 2020", car_parts=CarParts.common([CarHarness.subaru_b])),
   CAR.FORESTER_HYBRID: SubaruCarInfo("Subaru Forester Hybrid 2020"),
   CAR.FORESTER: SubaruCarInfo("Subaru Forester 2019-21", "All"),
   CAR.FORESTER_PREGLOBAL: SubaruCarInfo("Subaru Forester 2017-18"),
@@ -668,19 +668,23 @@ FW_VERSIONS = {
   },
   CAR.OUTBACK_2023: {
     (Ecu.abs, 0x7b0, None): [
+      b'\xa1 #\x14\x00',
       b'\xa1 #\x17\x00',
     ],
     (Ecu.eps, 0x746, None): [
+      b'+\xc0\x10\x11\x00',
       b'+\xc0\x12\x11\x00',
     ],
     (Ecu.fwdCamera, 0x787, None): [
       b'\t!\x08\x046\x05!\x08\x01/',
     ],
     (Ecu.engine, 0x7a2, None): [
+      b'\xed,\xa0q\x07',
       b'\xed,\xa2q\x07',
     ],
     (Ecu.transmission, 0x7a3, None): [
       b'\xa8\x8e\xf41\x00',
+      b'\xa8\xfe\xf41\x00',
     ]
   }
 }

selfdrive/car/tests/routes.py

@@ -180,27 +180,27 @@ routes = [
   CarTestRoute("54034823d30962f5|2021-05-24--06-37-34", TOYOTA.CAMRYH),
   CarTestRoute("4e45c89c38e8ec4d|2021-05-02--02-49-28", TOYOTA.COROLLA),
   CarTestRoute("5f5afb36036506e4|2019-05-14--02-09-54", TOYOTA.COROLLA_TSS2),
-  CarTestRoute("5ceff72287a5c86c|2019-10-19--10-59-02", TOYOTA.COROLLAH_TSS2),
+  CarTestRoute("5ceff72287a5c86c|2019-10-19--10-59-02", TOYOTA.COROLLA_TSS2),  # hybrid
   CarTestRoute("d2525c22173da58b|2021-04-25--16-47-04", TOYOTA.PRIUS),
   CarTestRoute("32a7df20486b0f70|2020-02-06--16-06-50", TOYOTA.RAV4H),
   CarTestRoute("cdf2f7de565d40ae|2019-04-25--03-53-41", TOYOTA.RAV4_TSS2),
   CarTestRoute("a5c341bb250ca2f0|2022-05-18--16-05-17", TOYOTA.RAV4_TSS2_2022),
-  CarTestRoute("7e34a988419b5307|2019-12-18--19-13-30", TOYOTA.RAV4H_TSS2),
+  CarTestRoute("7e34a988419b5307|2019-12-18--19-13-30", TOYOTA.RAV4_TSS2),  # hybrid
   CarTestRoute("2475fb3eb2ffcc2e|2022-04-29--12-46-23", TOYOTA.RAV4H_TSS2_2022),
   CarTestRoute("7a31f030957b9c85|2023-04-01--14-12-51", TOYOTA.LEXUS_ES),
   CarTestRoute("e6a24be49a6cd46e|2019-10-29--10-52-42", TOYOTA.LEXUS_ES_TSS2),
+  CarTestRoute("f49e8041283f2939|2019-05-30--11-51-51", TOYOTA.LEXUS_ES_TSS2),  # hybrid
   CarTestRoute("da23c367491f53e2|2021-05-21--09-09-11", TOYOTA.LEXUS_CTH, segment=3),
-  CarTestRoute("f49e8041283f2939|2019-05-30--11-51-51", TOYOTA.LEXUS_ESH_TSS2),
   CarTestRoute("37041c500fd30100|2020-12-30--12-17-24", TOYOTA.LEXUS_ESH),
   CarTestRoute("32696cea52831b02|2021-11-19--18-13-30", TOYOTA.LEXUS_RC),
   CarTestRoute("886fcd8408d570e9|2020-01-29--02-18-55", TOYOTA.LEXUS_RX),
   CarTestRoute("d27ad752e9b08d4f|2021-05-26--19-39-51", TOYOTA.LEXUS_RXH),
   CarTestRoute("01b22eb2ed121565|2020-02-02--11-25-51", TOYOTA.LEXUS_RX_TSS2),
-  CarTestRoute("b74758c690a49668|2020-05-20--15-58-57", TOYOTA.LEXUS_RXH_TSS2),
+  CarTestRoute("b74758c690a49668|2020-05-20--15-58-57", TOYOTA.LEXUS_RX_TSS2),  # hybrid
-  CarTestRoute("ec429c0f37564e3c|2020-02-01--17-28-12", TOYOTA.LEXUS_NXH),
   CarTestRoute("964c09eb11ca8089|2020-11-03--22-04-00", TOYOTA.LEXUS_NX),
+  CarTestRoute("ec429c0f37564e3c|2020-02-01--17-28-12", TOYOTA.LEXUS_NX),  # hybrid
   CarTestRoute("3fd5305f8b6ca765|2021-04-28--19-26-49", TOYOTA.LEXUS_NX_TSS2),
-  CarTestRoute("09ae96064ed85a14|2022-06-09--12-22-31", TOYOTA.LEXUS_NXH_TSS2),
+  CarTestRoute("09ae96064ed85a14|2022-06-09--12-22-31", TOYOTA.LEXUS_NX_TSS2),  # hybrid
   CarTestRoute("0a302ffddbb3e3d3|2020-02-08--16-19-08", TOYOTA.HIGHLANDER_TSS2),
   CarTestRoute("437e4d2402abf524|2021-05-25--07-58-50", TOYOTA.HIGHLANDERH_TSS2),
   CarTestRoute("3183cd9b021e89ce|2021-05-25--10-34-44", TOYOTA.HIGHLANDER),
@@ -211,9 +211,9 @@ routes = [
   CarTestRoute("0a0de17a1e6a2d15|2020-09-21--21-24-41", TOYOTA.PRIUS_TSS2),
   CarTestRoute("9b36accae406390e|2021-03-30--10-41-38", TOYOTA.MIRAI),
   CarTestRoute("cd9cff4b0b26c435|2021-05-13--15-12-39", TOYOTA.CHR),
+  CarTestRoute("57858ede0369a261|2021-05-18--20-34-20", TOYOTA.CHR),  # hybrid
   CarTestRoute("ea8fbe72b96a185c|2023-02-08--15-11-46", TOYOTA.CHR_TSS2),
   CarTestRoute("ea8fbe72b96a185c|2023-02-22--09-20-34", TOYOTA.CHR_TSS2),  # openpilot longitudinal, with smartDSU
-  CarTestRoute("57858ede0369a261|2021-05-18--20-34-20", TOYOTA.CHRH),
   CarTestRoute("6719965b0e1d1737|2023-02-09--22-44-05", TOYOTA.CHRH_TSS2),
   # CarTestRoute("6719965b0e1d1737|2023-08-29--06-40-05", TOYOTA.CHRH_TSS2),  # openpilot longitudinal, radar disabled
   CarTestRoute("14623aae37e549f3|2021-10-24--01-20-49", TOYOTA.PRIUS_V),

selfdrive/car/tests/test_fw_fingerprint.py

@@ -97,7 +97,7 @@ class TestFwFingerprint(unittest.TestCase):
 
   def test_fw_version_lists(self):
     for car_model, ecus in FW_VERSIONS.items():
-      with self.subTest(car_model=car_model):
+      with self.subTest(car_model=car_model.value):
         for ecu, ecu_fw in ecus.items():
           with self.subTest(ecu):
             duplicates = {fw for fw in ecu_fw if ecu_fw.count(fw) > 1}
@@ -119,13 +119,13 @@ class TestFwFingerprint(unittest.TestCase):
     for brand, config in FW_QUERY_CONFIGS.items():
       for car_model, ecus in VERSIONS[brand].items():
         bad_ecus = set(ecus).intersection(config.extra_ecus)
-        with self.subTest(car_model=car_model):
+        with self.subTest(car_model=car_model.value):
           self.assertFalse(len(bad_ecus), f'{car_model}: Fingerprints contain ECUs added for data collection: {bad_ecus}')
 
   def test_blacklisted_ecus(self):
     blacklisted_addrs = (0x7c4, 0x7d0)  # includes A/C ecu and an unknown ecu
     for car_model, ecus in FW_VERSIONS.items():
-      with self.subTest(car_model=car_model):
+      with self.subTest(car_model=car_model.value):
         CP = interfaces[car_model][0].get_non_essential_params(car_model)
         if CP.carName == 'subaru':
           for ecu in ecus.keys():

selfdrive/car/tests/test_models_segs.txt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b93de5a9a259934896d8573d9b0f696289b88693908968009fe144e2e10a3355
+oid sha256:2e57357c95937475fe92c13709ff2ee067f036643d5d455383bd9c3335525614
-size 127597
+size 126281

selfdrive/car/torque_data/params.yaml

@@ -45,15 +45,12 @@ KIA K5 2021: [2.405339728085138, 1.460032270828705, 0.11650989850813716]
 KIA NIRO EV 2020: [2.9215954981365337, 2.1500583840260044, 0.09236802474810267]
 KIA SORENTO GT LINE 2018: [2.464854685101844, 1.5335274218367956, 0.12056170567599558]
 KIA STINGER GT2 2018: [2.7499043387418967, 1.849652021986449, 0.12048334239559202]
-LEXUS ES 2019: [1.935835, 2.134803912579666, 0.093439]
+LEXUS ES 2019: [2.0357564999999997, 1.999082295195227, 0.101533]
-LEXUS ES HYBRID 2019: [2.135678, 1.863360677810788, 0.109627]
+LEXUS NX 2018: [2.3525924753753613, 1.9731412277641067, 0.15168101064205927]
-LEXUS NX 2018: [2.302625600642627, 2.1382378491466625, 0.14986840878892838]
 LEXUS NX 2020: [2.4331999786982936, 2.1045680431705414, 0.14099899317761067]
-LEXUS NX HYBRID 2018: [2.4025593501080955, 1.8080446063815507, 0.15349361249519017]
 LEXUS RX 2016: [1.5876816543130423, 1.0427699298523752, 0.21334066732397142]
-LEXUS RX 2020: [1.5228812994274734, 1.431102486563665, 0.164117]
+LEXUS RX 2020: [1.5375561442049257, 1.343166476215164, 0.1931062001527557]
 LEXUS RX HYBRID 2017: [1.6984261557042386, 1.3211501880159107, 0.1820354534928893]
-LEXUS RX HYBRID 2020: [1.5522309889823778, 1.255230465866663, 0.2220954003055114]
 MAZDA CX-9 2021: [1.7601682915983443, 1.0889677335154337, 0.17713792194297195]
 SKODA SUPERB 3RD GEN: [1.166437404652981, 1.1686163012668165, 0.12194533036948708]
 SUBARU FORESTER 2019: [3.6617001649776793, 2.342197172531713, 0.11075960785398745]
@@ -69,8 +66,7 @@ TOYOTA CAMRY 2021: [2.446083, 2.3476510120007434, 0.121615]
 TOYOTA CAMRY HYBRID 2018: [1.996333, 1.7996193116697359, 0.112565]
 TOYOTA CAMRY HYBRID 2021: [2.263582, 2.3901492458927986, 0.115257]
 TOYOTA COROLLA 2017: [3.117154369115421, 1.8438132575043773, 0.12289685869250652]
-TOYOTA COROLLA HYBRID TSS2 2019: [1.9079729107361805, 1.8118712531729109, 0.22251440891543514]
+TOYOTA COROLLA TSS2 2019: [1.991132339206426, 1.868866242720403, 0.19570063298031432]
-TOYOTA COROLLA TSS2 2019: [2.0742917676766712, 1.9258612322678952, 0.16888685704519352]
 TOYOTA HIGHLANDER 2017: [1.8696367437248915, 1.626293990451463, 0.17485372210240796]
 TOYOTA HIGHLANDER 2020: [2.022340166827233, 1.6183134804881791, 0.14592306380054457]
 TOYOTA HIGHLANDER HYBRID 2018: [1.752033, 1.6433903296845025, 0.144600]
@@ -79,13 +75,10 @@ TOYOTA MIRAI 2021: [2.506899832157829, 1.7417213930750164, 0.20182618449440565]
 TOYOTA PRIUS 2017: [1.60, 1.5023147650693636, 0.151515]
 TOYOTA PRIUS TSS2 2021: [1.972600, 1.9104337425537743, 0.170968]
 TOYOTA RAV4 2017: [2.085695074355425, 2.2142832316984733, 0.13339165270103975]
-TOYOTA RAV4 2019: [2.331293, 2.0993589721530252, 0.129822]
+TOYOTA RAV4 2019: [2.279239424615458, 2.087101966779332, 0.13682208413446817]
-TOYOTA RAV4 2019 8965: [2.5084506298290377, 2.4216520504763475, 0.11992835265067918]
+TOYOTA RAV4 2019 8965: [2.3080951748210854, 2.1189367835820603, 0.12942102328134028]
-TOYOTA RAV4 2019 x02: [2.7209621987605024, 2.2148637653781593, 0.10862567142268198]
+TOYOTA RAV4 2019 x02: [2.762293266024922, 2.243615865975329, 0.11113568178327986]
 TOYOTA RAV4 HYBRID 2017: [1.9796257271652042, 1.7503987331707576, 0.14628860048885406]
-TOYOTA RAV4 HYBRID 2019: [2.2271858492309153, 2.074844961405639, 0.14382216826893632]
-TOYOTA RAV4 HYBRID 2019 8965: [2.1077397198131336, 1.8162215166877735, 0.13891369391200137]
-TOYOTA RAV4 HYBRID 2019 x02: [2.803624333289342, 2.272367966572498, 0.11364569214387774]
 TOYOTA RAV4 HYBRID 2022: [2.241883248393209, 1.9304407208090029, 0.112174]
 TOYOTA RAV4 HYBRID 2022 x02: [3.044930631831037, 2.3979189796380918, 0.14023209146703736]
 TOYOTA SIENNA 2018: [1.689726, 1.3208264576110418, 0.140456]

selfdrive/car/torque_data/substitute.yaml

@@ -8,13 +8,11 @@ TOYOTA ALPHARD HYBRID 2021 : TOYOTA SIENNA 2018
 TOYOTA ALPHARD 2020: TOYOTA SIENNA 2018
 TOYOTA PRIUS v 2017 : TOYOTA PRIUS 2017
 TOYOTA RAV4 2022: TOYOTA RAV4 HYBRID 2022
-TOYOTA C-HR HYBRID 2018: TOYOTA C-HR 2018
 TOYOTA C-HR HYBRID 2022: TOYOTA C-HR 2021
 LEXUS IS 2018: LEXUS NX 2018
 LEXUS CT HYBRID 2018 : LEXUS NX 2018
 LEXUS ES 2018: TOYOTA CAMRY HYBRID 2018
 LEXUS ES HYBRID 2018: TOYOTA CAMRY HYBRID 2018
-LEXUS NX HYBRID 2020: LEXUS NX 2020
 LEXUS RC 2020: LEXUS NX 2020
 TOYOTA AVALON HYBRID 2019: TOYOTA AVALON 2019
 TOYOTA AVALON HYBRID 2022: TOYOTA AVALON 2022

selfdrive/car/toyota/carstate.py

@@ -58,9 +58,7 @@ class CarState(CarStateBase):
       ret.gas = (cp.vl["GAS_SENSOR"]["INTERCEPTOR_GAS"] + cp.vl["GAS_SENSOR"]["INTERCEPTOR_GAS2"]) // 2
       ret.gasPressed = ret.gas > 805
     else:
-      # TODO: find a new, common signal
+      # TODO: find a common gas pedal percentage signal
-      msg = "GAS_PEDAL_HYBRID" if (self.CP.flags & ToyotaFlags.HYBRID) else "GAS_PEDAL"
-      ret.gas = cp.vl[msg]["GAS_PEDAL"]
       ret.gasPressed = cp.vl["PCM_CRUISE"]["GAS_RELEASED"] == 0
 
     ret.wheelSpeeds = self.get_wheel_speeds(
@@ -183,11 +181,6 @@ class CarState(CarStateBase):
       ("STEER_TORQUE_SENSOR", 50),
     ]
 
-    if CP.flags & ToyotaFlags.HYBRID:
-      messages.append(("GAS_PEDAL_HYBRID", 33))
-    else:
-      messages.append(("GAS_PEDAL", 33))
-
     if CP.carFingerprint in UNSUPPORTED_DSU_CAR:
       messages.append(("DSU_CRUISE", 5))
       messages.append(("PCM_CRUISE_ALT", 1))

selfdrive/car/toyota/interface.py

@@ -3,7 +3,7 @@ from openpilot.common.conversions import Conversions as CV
 from panda import Panda
 from panda.python import uds
 from openpilot.selfdrive.car.toyota.values import Ecu, CAR, DBC, ToyotaFlags, CarControllerParams, TSS2_CAR, RADAR_ACC_CAR, NO_DSU_CAR, \
-                                        MIN_ACC_SPEED, EPS_SCALE, EV_HYBRID_CAR, UNSUPPORTED_DSU_CAR, NO_STOP_TIMER_CAR, ANGLE_CONTROL_CAR
+                                        MIN_ACC_SPEED, EPS_SCALE, UNSUPPORTED_DSU_CAR, NO_STOP_TIMER_CAR, ANGLE_CONTROL_CAR
 from openpilot.selfdrive.car import get_safety_config
 from openpilot.selfdrive.car.disable_ecu import disable_ecu
 from openpilot.selfdrive.car.interfaces import CarInterfaceBase
@@ -77,7 +77,7 @@ class CarInterface(CarInterfaceBase):
       ret.tireStiffnessFactor = 0.444  # not optimized yet
       ret.mass = 2860. * CV.LB_TO_KG  # mean between normal and hybrid
 
-    elif candidate in (CAR.LEXUS_RX, CAR.LEXUS_RXH, CAR.LEXUS_RX_TSS2, CAR.LEXUS_RXH_TSS2):
+    elif candidate in (CAR.LEXUS_RX, CAR.LEXUS_RXH, CAR.LEXUS_RX_TSS2):
       stop_and_go = True
       ret.wheelbase = 2.79
       ret.steerRatio = 16.  # 14.8 is spec end-to-end
@@ -85,7 +85,7 @@ class CarInterface(CarInterfaceBase):
       ret.tireStiffnessFactor = 0.5533
       ret.mass = 4481. * CV.LB_TO_KG  # mean between min and max
 
-    elif candidate in (CAR.CHR, CAR.CHRH, CAR.CHR_TSS2, CAR.CHRH_TSS2):
+    elif candidate in (CAR.CHR, CAR.CHR_TSS2, CAR.CHRH_TSS2):
       stop_and_go = True
       ret.wheelbase = 2.63906
       ret.steerRatio = 13.6
@@ -115,7 +115,7 @@ class CarInterface(CarInterfaceBase):
       ret.tireStiffnessFactor = 0.7983
       ret.mass = 3505. * CV.LB_TO_KG  # mean between normal and hybrid
 
-    elif candidate in (CAR.RAV4_TSS2, CAR.RAV4_TSS2_2022, CAR.RAV4H_TSS2, CAR.RAV4H_TSS2_2022,
+    elif candidate in (CAR.RAV4_TSS2, CAR.RAV4_TSS2_2022, CAR.RAV4H_TSS2_2022,
                        CAR.RAV4_TSS2_2023, CAR.RAV4H_TSS2_2023):
       ret.wheelbase = 2.68986
       ret.steerRatio = 14.3
@@ -137,13 +137,13 @@ class CarInterface(CarInterfaceBase):
           ret.lateralTuning.pid.kf = 0.00004
           break
 
-    elif candidate in (CAR.COROLLA_TSS2, CAR.COROLLAH_TSS2):
+    elif candidate == CAR.COROLLA_TSS2:
       ret.wheelbase = 2.67  # Average between 2.70 for sedan and 2.64 for hatchback
       ret.steerRatio = 13.9
       ret.tireStiffnessFactor = 0.444  # not optimized yet
       ret.mass = 3060. * CV.LB_TO_KG
 
-    elif candidate in (CAR.LEXUS_ES, CAR.LEXUS_ESH, CAR.LEXUS_ES_TSS2, CAR.LEXUS_ESH_TSS2):
+    elif candidate in (CAR.LEXUS_ES, CAR.LEXUS_ESH, CAR.LEXUS_ES_TSS2):
       if candidate not in (CAR.LEXUS_ES,):  # TODO: LEXUS_ES may have sng
         stop_and_go = True
       ret.wheelbase = 2.8702
@@ -171,7 +171,7 @@ class CarInterface(CarInterfaceBase):
       ret.tireStiffnessFactor = 0.517
       ret.mass = 3108 * CV.LB_TO_KG  # mean between min and max
 
-    elif candidate in (CAR.LEXUS_NX, CAR.LEXUS_NXH, CAR.LEXUS_NX_TSS2, CAR.LEXUS_NXH_TSS2):
+    elif candidate in (CAR.LEXUS_NX, CAR.LEXUS_NX_TSS2):
       stop_and_go = True
       ret.wheelbase = 2.66
       ret.steerRatio = 14.7
@@ -245,11 +245,6 @@ class CarInterface(CarInterfaceBase):
     if not ret.openpilotLongitudinalControl:
       ret.safetyConfigs[0].safetyParam |= Panda.FLAG_TOYOTA_STOCK_LONGITUDINAL
 
-    # we can't use the fingerprint to detect this reliably, since
-    # the EV gas pedal signal can take a couple seconds to appear
-    if candidate in EV_HYBRID_CAR:
-      ret.flags |= ToyotaFlags.HYBRID.value
-
     # min speed to enable ACC. if car can do stop and go, then set enabling speed
     # to a negative value, so it won't matter.
     ret.minEnableSpeed = -1. if (stop_and_go or ret.enableGasInterceptor) else MIN_ACC_SPEED

selfdrive/car/toyota/tests/test_toyota.py

@@ -3,8 +3,10 @@ from hypothesis import given, settings, strategies as st
 import unittest
 
 from cereal import car
+from openpilot.selfdrive.car.fw_versions import build_fw_dict
 from openpilot.selfdrive.car.toyota.values import CAR, DBC, TSS2_CAR, ANGLE_CONTROL_CAR, RADAR_ACC_CAR, FW_VERSIONS, \
-                                                  PLATFORM_CODE_ECUS, get_platform_codes
+                                                  FW_QUERY_CONFIG, PLATFORM_CODE_ECUS, FUZZY_EXCLUDED_PLATFORMS, \
+                                                  get_platform_codes
 
 Ecu = car.CarParams.Ecu
 ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()}
@@ -26,7 +28,7 @@ class TestToyotaInterfaces(unittest.TestCase):
     # Asserts standard ECUs exist for each platform
     common_ecus = {Ecu.fwdRadar, Ecu.fwdCamera}
     for car_model, ecus in FW_VERSIONS.items():
-      with self.subTest(car_model=car_model):
+      with self.subTest(car_model=car_model.value):
         present_ecus = {ecu[0] for ecu in ecus}
         missing_ecus = common_ecus - present_ecus
         self.assertEqual(len(missing_ecus), 0)
@@ -55,7 +57,7 @@ class TestToyotaFingerprint(unittest.TestCase):
   def test_platform_code_ecus_available(self):
     # Asserts ECU keys essential for fuzzy fingerprinting are available on all platforms
     for car_model, ecus in FW_VERSIONS.items():
-      with self.subTest(car_model=car_model):
+      with self.subTest(car_model=car_model.value):
         for platform_code_ecu in PLATFORM_CODE_ECUS:
           if platform_code_ecu == Ecu.eps and car_model in (CAR.PRIUS_V, CAR.LEXUS_CTH,):
             continue
@@ -70,7 +72,7 @@ class TestToyotaFingerprint(unittest.TestCase):
     # - expected parsing of ECU sub-versions
 
     for car_model, ecus in FW_VERSIONS.items():
-      with self.subTest(car_model=car_model):
+      with self.subTest(car_model=car_model.value):
         for ecu, fws in ecus.items():
           if ecu[0] not in PLATFORM_CODE_ECUS:
             continue
@@ -120,6 +122,27 @@ class TestToyotaFingerprint(unittest.TestCase):
     ])
     self.assertEqual(results, {b"F1526-07-1": {b"10", b"40"}, b"8646F-41-04": {b"100"}, b"58-79": {b"000"}})
 
+  def test_fuzzy_excluded_platforms(self):
+    # Asserts a list of platforms that will not fuzzy fingerprint with platform codes due to them being shared.
+    platforms_with_shared_codes = set()
+    for platform, fw_by_addr in FW_VERSIONS.items():
+      car_fw = []
+      for ecu, fw_versions in fw_by_addr.items():
+        ecu_name, addr, sub_addr = ecu
+        for fw in fw_versions:
+          car_fw.append({"ecu": ecu_name, "fwVersion": fw, "address": addr,
+                         "subAddress": 0 if sub_addr is None else sub_addr})
+
+      CP = car.CarParams.new_message(carFw=car_fw)
+      matches = FW_QUERY_CONFIG.match_fw_to_car_fuzzy(build_fw_dict(CP.carFw))
+      if len(matches) == 1:
+        self.assertEqual(list(matches)[0], platform)
+      else:
+        # If a platform has multiple matches, add it and its matches
+        platforms_with_shared_codes |= {platform, *matches}
+
+    self.assertEqual(platforms_with_shared_codes, FUZZY_EXCLUDED_PLATFORMS, (len(platforms_with_shared_codes), len(FW_VERSIONS)))
+
 
 if __name__ == "__main__":
   unittest.main()

selfdrive/car/toyota/values.py

@@ -61,12 +61,10 @@ class CAR(StrEnum):
   CAMRYH_TSS2 = "TOYOTA CAMRY HYBRID 2021"
   CHR = "TOYOTA C-HR 2018"
   CHR_TSS2 = "TOYOTA C-HR 2021"
-  CHRH = "TOYOTA C-HR HYBRID 2018"
   CHRH_TSS2 = "TOYOTA C-HR HYBRID 2022"
   COROLLA = "TOYOTA COROLLA 2017"
-  COROLLA_TSS2 = "TOYOTA COROLLA TSS2 2019"
   # LSS2 Lexus UX Hybrid is same as a TSS2 Corolla Hybrid
-  COROLLAH_TSS2 = "TOYOTA COROLLA HYBRID TSS2 2019"
+  COROLLA_TSS2 = "TOYOTA COROLLA TSS2 2019"
   HIGHLANDER = "TOYOTA HIGHLANDER 2017"
   HIGHLANDER_TSS2 = "TOYOTA HIGHLANDER 2020"
   HIGHLANDERH = "TOYOTA HIGHLANDER HYBRID 2018"
@@ -79,7 +77,6 @@ class CAR(StrEnum):
   RAV4_TSS2 = "TOYOTA RAV4 2019"
   RAV4_TSS2_2022 = "TOYOTA RAV4 2022"
   RAV4_TSS2_2023 = "TOYOTA RAV4 2023"
-  RAV4H_TSS2 = "TOYOTA RAV4 HYBRID 2019"
   RAV4H_TSS2_2022 = "TOYOTA RAV4 HYBRID 2022"
   RAV4H_TSS2_2023 = "TOYOTA RAV4 HYBRID 2023"
   MIRAI = "TOYOTA MIRAI 2021"  # TSS 2.5
@@ -90,18 +87,14 @@ class CAR(StrEnum):
   LEXUS_ES = "LEXUS ES 2018"
   LEXUS_ESH = "LEXUS ES HYBRID 2018"
   LEXUS_ES_TSS2 = "LEXUS ES 2019"
-  LEXUS_ESH_TSS2 = "LEXUS ES HYBRID 2019"
   LEXUS_IS = "LEXUS IS 2018"
   LEXUS_IS_TSS2 = "LEXUS IS 2023"
   LEXUS_NX = "LEXUS NX 2018"
-  LEXUS_NXH = "LEXUS NX HYBRID 2018"
   LEXUS_NX_TSS2 = "LEXUS NX 2020"
-  LEXUS_NXH_TSS2 = "LEXUS NX HYBRID 2020"
   LEXUS_RC = "LEXUS RC 2020"
   LEXUS_RX = "LEXUS RX 2016"
   LEXUS_RXH = "LEXUS RX HYBRID 2017"
   LEXUS_RX_TSS2 = "LEXUS RX 2020"
-  LEXUS_RXH_TSS2 = "LEXUS RX HYBRID 2020"
 
 
 class Footnote(Enum):
@@ -132,17 +125,18 @@ CAR_INFO: Dict[str, Union[ToyotaCarInfo, List[ToyotaCarInfo]]] = {
   CAR.CAMRYH: ToyotaCarInfo("Toyota Camry Hybrid 2018-20", video_link="https://www.youtube.com/watch?v=Q2DYY0AWKgk"),
   CAR.CAMRY_TSS2: ToyotaCarInfo("Toyota Camry 2021-23", footnotes=[Footnote.CAMRY]),
   CAR.CAMRYH_TSS2: ToyotaCarInfo("Toyota Camry Hybrid 2021-24"),
-  CAR.CHR: ToyotaCarInfo("Toyota C-HR 2017-20"),
+  CAR.CHR: [
+    ToyotaCarInfo("Toyota C-HR 2017-20"),
+    ToyotaCarInfo("Toyota C-HR Hybrid 2017-20"),
+  ],
   CAR.CHR_TSS2: ToyotaCarInfo("Toyota C-HR 2021"),
-  CAR.CHRH: ToyotaCarInfo("Toyota C-HR Hybrid 2017-20"),
   CAR.CHRH_TSS2: ToyotaCarInfo("Toyota C-HR Hybrid 2021-22"),
   CAR.COROLLA: ToyotaCarInfo("Toyota Corolla 2017-19"),
   CAR.COROLLA_TSS2: [
     ToyotaCarInfo("Toyota Corolla 2020-22", video_link="https://www.youtube.com/watch?v=_66pXk0CBYA"),
     ToyotaCarInfo("Toyota Corolla Cross (Non-US only) 2020-23", min_enable_speed=7.5),
     ToyotaCarInfo("Toyota Corolla Hatchback 2019-22", video_link="https://www.youtube.com/watch?v=_66pXk0CBYA"),
-  ],
+    # Hybrid platforms
-  CAR.COROLLAH_TSS2: [
     ToyotaCarInfo("Toyota Corolla Hybrid 2020-22"),
     ToyotaCarInfo("Toyota Corolla Hybrid (Non-US only) 2020-23", min_enable_speed=7.5),
     ToyotaCarInfo("Toyota Corolla Cross Hybrid (Non-US only) 2020-22", min_enable_speed=7.5),
@@ -170,10 +164,12 @@ CAR_INFO: Dict[str, Union[ToyotaCarInfo, List[ToyotaCarInfo]]] = {
     ToyotaCarInfo("Toyota RAV4 Hybrid 2016", "Toyota Safety Sense P", video_link="https://youtu.be/LhT5VzJVfNI?t=26"),
     ToyotaCarInfo("Toyota RAV4 Hybrid 2017-18", video_link="https://youtu.be/LhT5VzJVfNI?t=26")
   ],
-  CAR.RAV4_TSS2: ToyotaCarInfo("Toyota RAV4 2019-21", video_link="https://www.youtube.com/watch?v=wJxjDd42gGA"),
+  CAR.RAV4_TSS2: [
+    ToyotaCarInfo("Toyota RAV4 2019-21", video_link="https://www.youtube.com/watch?v=wJxjDd42gGA"),
+    ToyotaCarInfo("Toyota RAV4 Hybrid 2019-21"),
+  ],
   CAR.RAV4_TSS2_2022: ToyotaCarInfo("Toyota RAV4 2022"),
   CAR.RAV4_TSS2_2023: ToyotaCarInfo("Toyota RAV4 2023"),
-  CAR.RAV4H_TSS2: ToyotaCarInfo("Toyota RAV4 Hybrid 2019-21"),
   CAR.RAV4H_TSS2_2022: ToyotaCarInfo("Toyota RAV4 Hybrid 2022", video_link="https://youtu.be/U0nH9cnrFB0"),
   CAR.RAV4H_TSS2_2023: ToyotaCarInfo("Toyota RAV4 Hybrid 2023"),
   CAR.MIRAI: ToyotaCarInfo("Toyota Mirai 2021"),
@@ -183,14 +179,20 @@ CAR_INFO: Dict[str, Union[ToyotaCarInfo, List[ToyotaCarInfo]]] = {
   CAR.LEXUS_CTH: ToyotaCarInfo("Lexus CT Hybrid 2017-18", "Lexus Safety System+"),
   CAR.LEXUS_ES: ToyotaCarInfo("Lexus ES 2017-18"),
   CAR.LEXUS_ESH: ToyotaCarInfo("Lexus ES Hybrid 2017-18"),
-  CAR.LEXUS_ES_TSS2: ToyotaCarInfo("Lexus ES 2019-22"),
+  CAR.LEXUS_ES_TSS2: [
-  CAR.LEXUS_ESH_TSS2: ToyotaCarInfo("Lexus ES Hybrid 2019-23", video_link="https://youtu.be/BZ29osRVJeg?t=12"),
+    ToyotaCarInfo("Lexus ES 2019-24"),
+    ToyotaCarInfo("Lexus ES Hybrid 2019-23", video_link="https://youtu.be/BZ29osRVJeg?t=12"),
+  ],
   CAR.LEXUS_IS: ToyotaCarInfo("Lexus IS 2017-19"),
   CAR.LEXUS_IS_TSS2: ToyotaCarInfo("Lexus IS 2022-23"),
-  CAR.LEXUS_NX: ToyotaCarInfo("Lexus NX 2018-19"),
+  CAR.LEXUS_NX: [
-  CAR.LEXUS_NXH: ToyotaCarInfo("Lexus NX Hybrid 2018-19"),
+    ToyotaCarInfo("Lexus NX 2018-19"),
-  CAR.LEXUS_NX_TSS2: ToyotaCarInfo("Lexus NX 2020-21"),
+    ToyotaCarInfo("Lexus NX Hybrid 2018-19"),
-  CAR.LEXUS_NXH_TSS2: ToyotaCarInfo("Lexus NX Hybrid 2020-21"),
+  ],
+  CAR.LEXUS_NX_TSS2: [
+    ToyotaCarInfo("Lexus NX 2020-21"),
+    ToyotaCarInfo("Lexus NX Hybrid 2020-21"),
+  ],
   CAR.LEXUS_RC: ToyotaCarInfo("Lexus RC 2018-20"),
   CAR.LEXUS_RX: [
     ToyotaCarInfo("Lexus RX 2016", "Lexus Safety System+"),
@@ -200,38 +202,40 @@ CAR_INFO: Dict[str, Union[ToyotaCarInfo, List[ToyotaCarInfo]]] = {
     ToyotaCarInfo("Lexus RX Hybrid 2016", "Lexus Safety System+"),
     ToyotaCarInfo("Lexus RX Hybrid 2017-19"),
   ],
-  CAR.LEXUS_RX_TSS2: ToyotaCarInfo("Lexus RX 2020-22"),
+  CAR.LEXUS_RX_TSS2: [
-  CAR.LEXUS_RXH_TSS2: ToyotaCarInfo("Lexus RX Hybrid 2020-22"),
+    ToyotaCarInfo("Lexus RX 2020-22"),
+    ToyotaCarInfo("Lexus RX Hybrid 2020-22"),
+  ],
 }
 
 # (addr, cars, bus, 1/freq*100, vl)
 STATIC_DSU_MSGS = [
-  (0x128, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH, CAR.LEXUS_NXH, CAR.LEXUS_NX, CAR.RAV4, CAR.COROLLA, CAR.AVALON), 1,   3, b'\xf4\x01\x90\x83\x00\x37'),
+  (0x128, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH, CAR.LEXUS_NX, CAR.RAV4, CAR.COROLLA, CAR.AVALON), 1,   3, b'\xf4\x01\x90\x83\x00\x37'),
   (0x128, (CAR.HIGHLANDER, CAR.HIGHLANDERH, CAR.SIENNA, CAR.LEXUS_CTH, CAR.LEXUS_ES, CAR.LEXUS_ESH), 1,   3, b'\x03\x00\x20\x00\x00\x52'),
-  (0x141, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH, CAR.LEXUS_NXH, CAR.LEXUS_NX, CAR.RAV4, CAR.COROLLA, CAR.HIGHLANDER, CAR.HIGHLANDERH, CAR.AVALON,
+  (0x141, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH, CAR.LEXUS_NX, CAR.RAV4, CAR.COROLLA, CAR.HIGHLANDER, CAR.HIGHLANDERH, CAR.AVALON,
            CAR.SIENNA, CAR.LEXUS_CTH, CAR.LEXUS_ES, CAR.LEXUS_ESH, CAR.LEXUS_RX, CAR.PRIUS_V), 1,   2, b'\x00\x00\x00\x46'),
-  (0x160, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH, CAR.LEXUS_NXH, CAR.LEXUS_NX, CAR.RAV4, CAR.COROLLA, CAR.HIGHLANDER, CAR.HIGHLANDERH, CAR.AVALON,
+  (0x160, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH, CAR.LEXUS_NX, CAR.RAV4, CAR.COROLLA, CAR.HIGHLANDER, CAR.HIGHLANDERH, CAR.AVALON,
            CAR.SIENNA, CAR.LEXUS_CTH, CAR.LEXUS_ES, CAR.LEXUS_ESH, CAR.LEXUS_RX, CAR.PRIUS_V), 1,   7, b'\x00\x00\x08\x12\x01\x31\x9c\x51'),
-  (0x161, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH, CAR.LEXUS_NXH, CAR.LEXUS_NX, CAR.RAV4, CAR.COROLLA, CAR.AVALON, CAR.LEXUS_RX, CAR.PRIUS_V, CAR.LEXUS_ES),
+  (0x161, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH, CAR.LEXUS_NX, CAR.RAV4, CAR.COROLLA, CAR.AVALON, CAR.LEXUS_RX, CAR.PRIUS_V, CAR.LEXUS_ES),
                                                                                                1,   7, b'\x00\x1e\x00\x00\x00\x80\x07'),
   (0X161, (CAR.HIGHLANDERH, CAR.HIGHLANDER, CAR.SIENNA, CAR.LEXUS_CTH, CAR.LEXUS_ESH), 1,  7, b'\x00\x1e\x00\xd4\x00\x00\x5b'),
-  (0x283, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH, CAR.LEXUS_NXH, CAR.LEXUS_NX, CAR.RAV4, CAR.COROLLA, CAR.HIGHLANDER, CAR.HIGHLANDERH, CAR.AVALON,
+  (0x283, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH, CAR.LEXUS_NX, CAR.RAV4, CAR.COROLLA, CAR.HIGHLANDER, CAR.HIGHLANDERH, CAR.AVALON,
            CAR.SIENNA, CAR.LEXUS_CTH, CAR.LEXUS_ES, CAR.LEXUS_ESH, CAR.LEXUS_RX, CAR.PRIUS_V), 0,   3, b'\x00\x00\x00\x00\x00\x00\x8c'),
   (0x2E6, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH), 0,   3, b'\xff\xf8\x00\x08\x7f\xe0\x00\x4e'),
   (0x2E7, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH), 0,   3, b'\xa8\x9c\x31\x9c\x00\x00\x00\x02'),
   (0x33E, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH), 0,  20, b'\x0f\xff\x26\x40\x00\x1f\x00'),
-  (0x344, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH, CAR.LEXUS_NXH, CAR.LEXUS_NX, CAR.RAV4, CAR.COROLLA, CAR.HIGHLANDER, CAR.HIGHLANDERH, CAR.AVALON,
+  (0x344, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH, CAR.LEXUS_NX, CAR.RAV4, CAR.COROLLA, CAR.HIGHLANDER, CAR.HIGHLANDERH, CAR.AVALON,
            CAR.SIENNA, CAR.LEXUS_CTH, CAR.LEXUS_ES, CAR.LEXUS_ESH, CAR.LEXUS_RX, CAR.PRIUS_V), 0,   5, b'\x00\x00\x01\x00\x00\x00\x00\x50'),
-  (0x365, (CAR.PRIUS, CAR.LEXUS_RXH, CAR.LEXUS_NXH, CAR.LEXUS_NX, CAR.HIGHLANDERH), 0,  20, b'\x00\x00\x00\x80\x03\x00\x08'),
+  (0x365, (CAR.PRIUS, CAR.LEXUS_RXH, CAR.LEXUS_NX, CAR.HIGHLANDERH), 0,  20, b'\x00\x00\x00\x80\x03\x00\x08'),
   (0x365, (CAR.RAV4, CAR.RAV4H, CAR.COROLLA, CAR.HIGHLANDER, CAR.AVALON, CAR.SIENNA, CAR.LEXUS_CTH, CAR.LEXUS_ES, CAR.LEXUS_ESH, CAR.LEXUS_RX,
            CAR.PRIUS_V), 0,  20, b'\x00\x00\x00\x80\xfc\x00\x08'),
-  (0x366, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH, CAR.LEXUS_NXH, CAR.LEXUS_NX, CAR.HIGHLANDERH), 0,  20, b'\x00\x00\x4d\x82\x40\x02\x00'),
+  (0x366, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH, CAR.LEXUS_NX, CAR.HIGHLANDERH), 0,  20, b'\x00\x00\x4d\x82\x40\x02\x00'),
   (0x366, (CAR.RAV4, CAR.COROLLA, CAR.HIGHLANDER, CAR.AVALON, CAR.SIENNA, CAR.LEXUS_CTH, CAR.LEXUS_ESH, CAR.LEXUS_RX, CAR.PRIUS_V),
           0,  20, b'\x00\x72\x07\xff\x09\xfe\x00'),
   (0x366, (CAR.LEXUS_ES,), 0,  20, b'\x00\x95\x07\xfe\x08\x05\x00'),
   (0x470, (CAR.PRIUS, CAR.LEXUS_RXH), 1, 100, b'\x00\x00\x02\x7a'),
   (0x470, (CAR.HIGHLANDER, CAR.HIGHLANDERH, CAR.RAV4H, CAR.SIENNA, CAR.LEXUS_CTH, CAR.LEXUS_ES, CAR.LEXUS_ESH, CAR.PRIUS_V), 1,  100, b'\x00\x00\x01\x79'),
-  (0x4CB, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH, CAR.LEXUS_NXH, CAR.LEXUS_NX, CAR.RAV4, CAR.COROLLA, CAR.HIGHLANDERH, CAR.HIGHLANDER, CAR.AVALON,
+  (0x4CB, (CAR.PRIUS, CAR.RAV4H, CAR.LEXUS_RXH, CAR.LEXUS_NX, CAR.RAV4, CAR.COROLLA, CAR.HIGHLANDERH, CAR.HIGHLANDER, CAR.AVALON,
            CAR.SIENNA, CAR.LEXUS_CTH, CAR.LEXUS_ES, CAR.LEXUS_ESH, CAR.LEXUS_RX, CAR.PRIUS_V), 0, 100, b'\x0c\x00\x00\x00\x00\x00\x00\x00'),
 ]
 
@@ -280,6 +284,40 @@ def get_platform_codes(fw_versions: List[bytes]) -> Dict[bytes, Set[bytes]]:
   return dict(codes)
 
 
+def match_fw_to_car_fuzzy(live_fw_versions) -> Set[str]:
+  candidates = set()
+
+  for candidate, fws in FW_VERSIONS.items():
+    # Keep track of ECUs which pass all checks (platform codes, within sub-version range)
+    valid_found_ecus = set()
+    valid_expected_ecus = {ecu[1:] for ecu in fws if ecu[0] in PLATFORM_CODE_ECUS}
+    for ecu, expected_versions in fws.items():
+      addr = ecu[1:]
+      # Only check ECUs expected to have platform codes
+      if ecu[0] not in PLATFORM_CODE_ECUS:
+        continue
+
+      # Expected platform codes & versions
+      expected_platform_codes = get_platform_codes(expected_versions)
+
+      # Found platform codes & versions
+      found_platform_codes = get_platform_codes(live_fw_versions.get(addr, set()))
+
+      # Check part number + platform code + major version matches for any found versions
+      # Platform codes and major versions change for different physical parts, generation, API, etc.
+      # Sub-versions are incremented for minor recalls, do not need to be checked.
+      if not any(found_platform_code in expected_platform_codes for found_platform_code in found_platform_codes):
+        break
+
+      valid_found_ecus.add(addr)
+
+    # If all live ECUs pass all checks for candidate, add it as a match
+    if valid_expected_ecus.issubset(valid_found_ecus):
+      candidates.add(candidate)
+
+  return {str(c) for c in (candidates - FUZZY_EXCLUDED_PLATFORMS)}
+
+
 # Regex patterns for parsing more general platform-specific identifiers from FW versions.
 # - Part number: Toyota part number (usually last character needs to be ignored to find a match).
 #    Each ECU address has just one part number.
@@ -307,6 +345,8 @@ FW_CHUNK_LEN = 16
 # - eps: describes lateral API changes for the EPS, such as using LTA for lane keeping and rejecting LKA messages
 PLATFORM_CODE_ECUS = [Ecu.fwdCamera, Ecu.abs, Ecu.eps]
 
+# These platforms have at least one platform code for all ECUs shared with another platform.
+FUZZY_EXCLUDED_PLATFORMS = {CAR.LEXUS_ES_TSS2, CAR.LEXUS_RX_TSS2}
 
 # Some ECUs that use KWP2000 have their FW versions on non-standard data identifiers.
 # Toyota diagnostic software first gets the supported data ids, then queries them one by one.
@@ -321,7 +361,7 @@ FW_QUERY_CONFIG = FwQueryConfig(
       [StdQueries.SHORT_TESTER_PRESENT_REQUEST, TOYOTA_VERSION_REQUEST_KWP],
       [StdQueries.SHORT_TESTER_PRESENT_RESPONSE, TOYOTA_VERSION_RESPONSE_KWP],
       whitelist_ecus=[Ecu.fwdCamera, Ecu.fwdRadar, Ecu.dsu, Ecu.abs, Ecu.eps, Ecu.epb, Ecu.telematics,
-                      Ecu.hybrid, Ecu.srs, Ecu.combinationMeter, Ecu.transmission, Ecu.gateway, Ecu.hvac],
+                      Ecu.srs, Ecu.combinationMeter, Ecu.transmission, Ecu.gateway, Ecu.hvac],
       bus=0,
     ),
     Request(
@@ -343,7 +383,8 @@ FW_QUERY_CONFIG = FwQueryConfig(
     # FIXME: On some models, abs can sometimes be missing
     Ecu.abs: [CAR.RAV4, CAR.COROLLA, CAR.HIGHLANDER, CAR.SIENNA, CAR.LEXUS_IS],
     # On some models, the engine can show on two different addresses
-    Ecu.engine: [CAR.CAMRY, CAR.COROLLA_TSS2, CAR.CHR, CAR.CHR_TSS2, CAR.LEXUS_IS, CAR.LEXUS_RC],
+    Ecu.engine: [CAR.CAMRY, CAR.COROLLA_TSS2, CAR.CHR, CAR.CHR_TSS2, CAR.LEXUS_IS, CAR.LEXUS_RC,
+                 CAR.LEXUS_NX, CAR.LEXUS_NX_TSS2, CAR.LEXUS_RX_TSS2],
   },
   extra_ecus=[
     # All known ECUs on a late-model Toyota vehicle not queried here:
@@ -377,6 +418,7 @@ FW_QUERY_CONFIG = FwQueryConfig(
     (Ecu.combinationMeter, 0x7c0, None),
     (Ecu.hvac, 0x7c4, None),
   ],
+  match_fw_to_car_fuzzy=match_fw_to_car_fuzzy,
 )
 
 FW_VERSIONS = {
@@ -590,6 +632,7 @@ FW_VERSIONS = {
       b'\x018966333X0000\x00\x00\x00\x00',
       b'\x018966333X4000\x00\x00\x00\x00',
       b'\x01896633T16000\x00\x00\x00\x00',
+      b'\x018966306L9000\x00\x00\x00\x00',
       b'\x028966306B2100\x00\x00\x00\x00897CF3302002\x00\x00\x00\x00',
       b'\x028966306B2300\x00\x00\x00\x00897CF3302002\x00\x00\x00\x00',
       b'\x028966306B2500\x00\x00\x00\x00897CF3302002\x00\x00\x00\x00',
@@ -738,6 +781,13 @@ FW_VERSIONS = {
       b'\x01896631017200\x00\x00\x00\x00',
       b'\x0189663F413100\x00\x00\x00\x00',
       b'\x0189663F414100\x00\x00\x00\x00',
+      b'\x0289663F405100\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x02896631013200\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x0289663F405000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x0289663F418000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x0289663F423000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x0289663F431000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x0189663F438000\x00\x00\x00\x00',
     ],
     (Ecu.dsu, 0x791, None): [
       b'8821F0W01000    ',
@@ -748,6 +798,9 @@ FW_VERSIONS = {
       b'8821FF405100    ',
       b'8821FF406000    ',
       b'8821FF407100    ',
+      b'8821FF402300    ',
+      b'8821FF402400    ',
+      b'8821FF405000    ',
     ],
     (Ecu.abs, 0x7b0, None): [
       b'F152610020\x00\x00\x00\x00\x00\x00',
@@ -758,11 +811,21 @@ FW_VERSIONS = {
       b'F1526F4073\x00\x00\x00\x00\x00\x00',
       b'F1526F4121\x00\x00\x00\x00\x00\x00',
       b'F1526F4122\x00\x00\x00\x00\x00\x00',
+      b'F152610012\x00\x00\x00\x00\x00\x00',
+      b'F152610013\x00\x00\x00\x00\x00\x00',
+      b'F152610014\x00\x00\x00\x00\x00\x00',
+      b'F152610040\x00\x00\x00\x00\x00\x00',
+      b'F152610190\x00\x00\x00\x00\x00\x00',
+      b'F152610200\x00\x00\x00\x00\x00\x00',
+      b'F152610220\x00\x00\x00\x00\x00\x00',
+      b'F152610230\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.eps, 0x7a1, None): [
       b'8965B10011\x00\x00\x00\x00\x00\x00',
       b'8965B10040\x00\x00\x00\x00\x00\x00',
       b'8965B10070\x00\x00\x00\x00\x00\x00',
+      b'8965B10020\x00\x00\x00\x00\x00\x00',
+      b'8965B10050\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.engine, 0x7e0, None): [
       b'\x0331024000\x00\x00\x00\x00\x00\x00\x00\x00A0202000\x00\x00\x00\x00\x00\x00\x00\x00895231203202\x00\x00\x00\x00',
@@ -782,6 +845,9 @@ FW_VERSIONS = {
       b'8821FF406000    ',
       b'8821FF407100    ',
       b'8821F0W01100    ',
+      b'8821FF402300    ',
+      b'8821FF402400    ',
+      b'8821FF405000    ',
     ],
     (Ecu.fwdCamera, 0x750, 0x6d): [
       b'8646FF401700    ',
@@ -789,6 +855,8 @@ FW_VERSIONS = {
       b'8646FF404000    ',
       b'8646FF406000    ',
       b'8646FF407000    ',
+      b'8646FF402100    ',
+      b'8646FF407100    ',
     ],
   },
   CAR.CHR_TSS2: {
@@ -815,61 +883,6 @@ FW_VERSIONS = {
       b'\x028646FF411100\x00\x00\x00\x008646GF409000\x00\x00\x00\x00',
     ],
   },
-  CAR.CHRH: {
-    (Ecu.engine, 0x700, None): [
-      b'\x0289663F405100\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x02896631013200\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x0289663F405000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x0289663F418000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x0289663F423000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x0289663F431000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x0189663F438000\x00\x00\x00\x00',
-    ],
-    (Ecu.abs, 0x7b0, None): [
-      b'F152610012\x00\x00\x00\x00\x00\x00',
-      b'F152610013\x00\x00\x00\x00\x00\x00',
-      b'F152610014\x00\x00\x00\x00\x00\x00',
-      b'F152610040\x00\x00\x00\x00\x00\x00',
-      b'F152610190\x00\x00\x00\x00\x00\x00',
-      b'F152610200\x00\x00\x00\x00\x00\x00',
-      b'F152610220\x00\x00\x00\x00\x00\x00',
-      b'F152610230\x00\x00\x00\x00\x00\x00',
-    ],
-    (Ecu.dsu, 0x791, None): [
-      b'8821F0W01000    ',
-      b'8821FF402300    ',
-      b'8821FF402400    ',
-      b'8821FF404000    ',
-      b'8821FF404100    ',
-      b'8821FF405000    ',
-      b'8821FF406000    ',
-      b'8821FF407100    ',
-    ],
-    (Ecu.eps, 0x7a1, None): [
-      b'8965B10011\x00\x00\x00\x00\x00\x00',
-      b'8965B10020\x00\x00\x00\x00\x00\x00',
-      b'8965B10040\x00\x00\x00\x00\x00\x00',
-      b'8965B10050\x00\x00\x00\x00\x00\x00',
-    ],
-    (Ecu.fwdRadar, 0x750, 0xf): [
-      b'8821F0W01000    ',
-      b'8821FF402300    ',
-      b'8821FF402400    ',
-      b'8821FF404000    ',
-      b'8821FF404100    ',
-      b'8821FF405000    ',
-      b'8821FF406000    ',
-      b'8821FF407100    ',
-    ],
-    (Ecu.fwdCamera, 0x750, 0x6d): [
-      b'8646FF401700    ',
-      b'8646FF402100    ',
-      b'8646FF404000    ',
-      b'8646FF406000    ',
-      b'8646FF407000    ',
-      b'8646FF407100    ',
-    ],
-  },
   CAR.CHRH_TSS2: {
     (Ecu.eps, 0x7a1, None): [
       b'8965B10092\x00\x00\x00\x00\x00\x00',
@@ -960,6 +973,33 @@ FW_VERSIONS = {
       b'\x018966312W3000\x00\x00\x00\x00',
       b'\x018966312W9000\x00\x00\x00\x00',
       b'\x01896637644000\x00\x00\x00\x00',
+      b'\x01896630ZJ1000\x00\x00\x00\x00',
+      b'\x01896630ZU8000\x00\x00\x00\x00',
+      b'\x01896637621000\x00\x00\x00\x00',
+      b'\x01896637623000\x00\x00\x00\x00',
+      b'\x01896637624000\x00\x00\x00\x00',
+      b'\x01896637626000\x00\x00\x00\x00',
+      b'\x01896637639000\x00\x00\x00\x00',
+      b'\x01896637648000\x00\x00\x00\x00',
+      b'\x01896637643000\x00\x00\x00\x00',
+      b'\x02896630A07000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x02896630A21000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x02896630ZJ5000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x02896630ZK8000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x02896630ZN8000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x02896630ZQ3000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x02896630ZR2000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x02896630ZT8000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x02896630ZT9000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x02896630ZZ0000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x028966312K6000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x028966312L0000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x028966312Q3000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x028966312Q3100\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x028966312Q4000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
+      b'\x038966312L7000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00897CF1205001\x00\x00\x00\x00',
+      b'\x038966312N1000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00897CF1203001\x00\x00\x00\x00',
+      b'\x038966312T3000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00897CF1205001\x00\x00\x00\x00',
     ],
     (Ecu.engine, 0x7e0, None): [
       b'\x0230A10000\x00\x00\x00\x00\x00\x00\x00\x00A0202000\x00\x00\x00\x00\x00\x00\x00\x00',
@@ -991,13 +1031,18 @@ FW_VERSIONS = {
       b'8965B76012\x00\x00\x00\x00\x00\x00',
       b'\x018965B12510\x00\x00\x00\x00\x00\x00',
       b'\x018965B1256000\x00\x00\x00\x00',
+      b'8965B12451\x00\x00\x00\x00\x00\x00',
+      b'8965B16101\x00\x00\x00\x00\x00\x00',
+      b'8965B16170\x00\x00\x00\x00\x00\x00',
+      b'8965B76050\x00\x00\x00\x00\x00\x00',
+      b'8965B76091\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.abs, 0x7b0, None): [
       b'\x01F152602280\x00\x00\x00\x00\x00\x00',
       b'\x01F152602560\x00\x00\x00\x00\x00\x00',
       b'\x01F152602590\x00\x00\x00\x00\x00\x00',
       b'\x01F152602650\x00\x00\x00\x00\x00\x00',
-      b"\x01F15260A010\x00\x00\x00\x00\x00\x00",
+      b'\x01F15260A010\x00\x00\x00\x00\x00\x00',
       b'\x01F15260A050\x00\x00\x00\x00\x00\x00',
       b'\x01F152612641\x00\x00\x00\x00\x00\x00',
       b'\x01F152612651\x00\x00\x00\x00\x00\x00',
@@ -1015,74 +1060,6 @@ FW_VERSIONS = {
       b'\x01F152612B91\x00\x00\x00\x00\x00\x00',
       b'\x01F15260A070\x00\x00\x00\x00\x00\x00',
       b'\x01F152676250\x00\x00\x00\x00\x00\x00',
-    ],
-    (Ecu.fwdRadar, 0x750, 0xf): [
-      b'\x018821F3301100\x00\x00\x00\x00',
-      b'\x018821F3301200\x00\x00\x00\x00',
-      b'\x018821F3301300\x00\x00\x00\x00',
-      b'\x018821F3301400\x00\x00\x00\x00',
-    ],
-    (Ecu.fwdCamera, 0x750, 0x6d): [
-      b'\x028646F12010D0\x00\x00\x00\x008646G26011A0\x00\x00\x00\x00',
-      b'\x028646F1201100\x00\x00\x00\x008646G26011A0\x00\x00\x00\x00',
-      b'\x028646F1201200\x00\x00\x00\x008646G26011A0\x00\x00\x00\x00',
-      b'\x028646F1201300\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
-      b'\x028646F1201400\x00\x00\x00\x008646G2601500\x00\x00\x00\x00',
-      b'\x028646F1202000\x00\x00\x00\x008646G2601200\x00\x00\x00\x00',
-      b'\x028646F1202100\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
-      b'\x028646F1202200\x00\x00\x00\x008646G2601500\x00\x00\x00\x00',
-      b'\x028646F1601100\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
-      b'\x028646F1601300\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
-    ],
-  },
-  CAR.COROLLAH_TSS2: {
-    (Ecu.engine, 0x700, None): [
-      b'\x01896630ZJ1000\x00\x00\x00\x00',
-      b'\x01896630ZU8000\x00\x00\x00\x00',
-      b'\x01896637621000\x00\x00\x00\x00',
-      b'\x01896637623000\x00\x00\x00\x00',
-      b'\x01896637624000\x00\x00\x00\x00',
-      b'\x01896637626000\x00\x00\x00\x00',
-      b'\x01896637639000\x00\x00\x00\x00',
-      b'\x01896637648000\x00\x00\x00\x00',
-      b'\x01896637643000\x00\x00\x00\x00',
-      b'\x02896630A07000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x02896630A21000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x02896630ZJ5000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x02896630ZK8000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x02896630ZN8000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x02896630ZQ3000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x02896630ZR2000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x02896630ZT8000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x02896630ZT9000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x02896630ZZ0000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x028966312K6000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x028966312L0000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x028966312Q3000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x028966312Q3100\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x028966312Q4000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00',
-      b'\x038966312L7000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00897CF1205001\x00\x00\x00\x00',
-      b'\x038966312N1000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00897CF1203001\x00\x00\x00\x00',
-      b'\x038966312T3000\x00\x00\x00\x008966A4703000\x00\x00\x00\x00897CF1205001\x00\x00\x00\x00',
-    ],
-    (Ecu.eps, 0x7a1, None): [
-      b'8965B12361\x00\x00\x00\x00\x00\x00',
-      b'8965B12451\x00\x00\x00\x00\x00\x00',
-      b'8965B16011\x00\x00\x00\x00\x00\x00',
-      b'8965B16101\x00\x00\x00\x00\x00\x00',
-      b'8965B16170\x00\x00\x00\x00\x00\x00',
-      b'8965B76012\x00\x00\x00\x00\x00\x00',
-      b'8965B76050\x00\x00\x00\x00\x00\x00',
-      b'8965B76091\x00\x00\x00\x00\x00\x00',
-      b'\x018965B12350\x00\x00\x00\x00\x00\x00',
-      b'\x018965B12470\x00\x00\x00\x00\x00\x00',
-      b'\x018965B12490\x00\x00\x00\x00\x00\x00',
-      b'\x018965B12500\x00\x00\x00\x00\x00\x00',
-      b'\x018965B12510\x00\x00\x00\x00\x00\x00',
-      b'\x018965B12520\x00\x00\x00\x00\x00\x00',
-      b'\x018965B12530\x00\x00\x00\x00\x00\x00',
-    ],
-    (Ecu.abs, 0x7b0, None): [
       b'F152612590\x00\x00\x00\x00\x00\x00',
       b'F152612691\x00\x00\x00\x00\x00\x00',
       b'F152612692\x00\x00\x00\x00\x00\x00',
@@ -1116,14 +1093,15 @@ FW_VERSIONS = {
     (Ecu.fwdCamera, 0x750, 0x6d): [
       b'\x028646F12010D0\x00\x00\x00\x008646G26011A0\x00\x00\x00\x00',
       b'\x028646F1201100\x00\x00\x00\x008646G26011A0\x00\x00\x00\x00',
+      b'\x028646F1201200\x00\x00\x00\x008646G26011A0\x00\x00\x00\x00',
       b'\x028646F1201300\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
       b'\x028646F1201400\x00\x00\x00\x008646G2601500\x00\x00\x00\x00',
       b'\x028646F1202000\x00\x00\x00\x008646G2601200\x00\x00\x00\x00',
       b'\x028646F1202100\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
       b'\x028646F1202200\x00\x00\x00\x008646G2601500\x00\x00\x00\x00',
       b'\x028646F1601100\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
+      b'\x028646F1601300\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
       b'\x028646F1601200\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
-      b"\x028646F1601300\x00\x00\x00\x008646G2601400\x00\x00\x00\x00",
       b'\x028646F4203400\x00\x00\x00\x008646G2601200\x00\x00\x00\x00',
       b'\x028646F76020C0\x00\x00\x00\x008646G26011A0\x00\x00\x00\x00',
       b'\x028646F7603100\x00\x00\x00\x008646G2601200\x00\x00\x00\x00',
@@ -1578,6 +1556,24 @@ FW_VERSIONS = {
       b'\x02896634A43000\x00\x00\x00\x00897CF4201001\x00\x00\x00\x00',
       b'\x02896634A47000\x00\x00\x00\x00897CF4201001\x00\x00\x00\x00',
       b'\x028966342Z8000\x00\x00\x00\x00897CF1201001\x00\x00\x00\x00',
+      b'\x01896634A15000\x00\x00\x00\x00',
+      b'\x018966342M5000\x00\x00\x00\x00',
+      b'\x018966342W8000\x00\x00\x00\x00',
+      b'\x018966342X5000\x00\x00\x00\x00',
+      b'\x018966342X6000\x00\x00\x00\x00',
+      b'\x01896634A25000\x00\x00\x00\x00',
+      b'\x018966342W5000\x00\x00\x00\x00',
+      b'\x018966342W7000\x00\x00\x00\x00',
+      b'\x028966342W4001\x00\x00\x00\x00897CF1203001\x00\x00\x00\x00',
+      b'\x02896634A13000\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00',
+      b'\x02896634A13001\x00\x00\x00\x00897CF4801001\x00\x00\x00\x00',
+      b'\x02896634A13101\x00\x00\x00\x00897CF4801001\x00\x00\x00\x00',
+      b'\x02896634A14001\x00\x00\x00\x00897CF1203001\x00\x00\x00\x00',
+      b'\x02896634A23000\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00',
+      b'\x02896634A23001\x00\x00\x00\x00897CF1203001\x00\x00\x00\x00',
+      b'\x02896634A23101\x00\x00\x00\x00897CF1203001\x00\x00\x00\x00',
+      b'\x02896634A14001\x00\x00\x00\x00897CF4801001\x00\x00\x00\x00',
+      b'\x02896634A14101\x00\x00\x00\x00897CF4801001\x00\x00\x00\x00',
     ],
     (Ecu.abs, 0x7b0, None): [
       b'\x01F15260R210\x00\x00\x00\x00\x00\x00',
@@ -1595,6 +1591,18 @@ FW_VERSIONS = {
       b'\x01F152642750\x00\x00\x00\x00\x00\x00',
       b'\x01F152642751\x00\x00\x00\x00\x00\x00',
       b'\x01F15260R292\x00\x00\x00\x00\x00\x00',
+      b'F152642291\x00\x00\x00\x00\x00\x00',
+      b'F152642290\x00\x00\x00\x00\x00\x00',
+      b'F152642322\x00\x00\x00\x00\x00\x00',
+      b'F152642330\x00\x00\x00\x00\x00\x00',
+      b'F152642331\x00\x00\x00\x00\x00\x00',
+      b'F152642531\x00\x00\x00\x00\x00\x00',
+      b'F152642532\x00\x00\x00\x00\x00\x00',
+      b'F152642520\x00\x00\x00\x00\x00\x00',
+      b'F152642521\x00\x00\x00\x00\x00\x00',
+      b'F152642540\x00\x00\x00\x00\x00\x00',
+      b'F152642541\x00\x00\x00\x00\x00\x00',
+      b'F152642542\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.eps, 0x7a1, None): [
       b'8965B42170\x00\x00\x00\x00\x00\x00',
@@ -1662,65 +1670,6 @@ FW_VERSIONS = {
       b'\x028646F0R05100\x00\x00\x00\x008646G0R02100\x00\x00\x00\x00',
     ],
   },
-  CAR.RAV4H_TSS2: {
-    (Ecu.engine, 0x700, None): [
-      b'\x01896634A15000\x00\x00\x00\x00',
-      b'\x018966342M5000\x00\x00\x00\x00',
-      b'\x018966342W8000\x00\x00\x00\x00',
-      b'\x018966342X5000\x00\x00\x00\x00',
-      b'\x018966342X6000\x00\x00\x00\x00',
-      b'\x01896634A25000\x00\x00\x00\x00',
-      b'\x018966342W5000\x00\x00\x00\x00',
-      b'\x018966342W7000\x00\x00\x00\x00',
-      b'\x028966342W4001\x00\x00\x00\x00897CF1203001\x00\x00\x00\x00',
-      b'\x02896634A13000\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00',
-      b'\x02896634A13001\x00\x00\x00\x00897CF4801001\x00\x00\x00\x00',
-      b'\x02896634A13101\x00\x00\x00\x00897CF4801001\x00\x00\x00\x00',
-      b'\x02896634A14001\x00\x00\x00\x00897CF1203001\x00\x00\x00\x00',
-      b'\x02896634A23000\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00',
-      b'\x02896634A23001\x00\x00\x00\x00897CF1203001\x00\x00\x00\x00',
-      b'\x02896634A23101\x00\x00\x00\x00897CF1203001\x00\x00\x00\x00',
-      b'\x02896634A14001\x00\x00\x00\x00897CF4801001\x00\x00\x00\x00',
-      b'\x02896634A14101\x00\x00\x00\x00897CF4801001\x00\x00\x00\x00',
-    ],
-    (Ecu.abs, 0x7b0, None): [
-      b'F152642291\x00\x00\x00\x00\x00\x00',
-      b'F152642290\x00\x00\x00\x00\x00\x00',
-      b'F152642322\x00\x00\x00\x00\x00\x00',
-      b'F152642330\x00\x00\x00\x00\x00\x00',
-      b'F152642331\x00\x00\x00\x00\x00\x00',
-      b'F152642531\x00\x00\x00\x00\x00\x00',
-      b'F152642532\x00\x00\x00\x00\x00\x00',
-      b'F152642520\x00\x00\x00\x00\x00\x00',
-      b'F152642521\x00\x00\x00\x00\x00\x00',
-      b'F152642540\x00\x00\x00\x00\x00\x00',
-      b'F152642541\x00\x00\x00\x00\x00\x00',
-      b'F152642542\x00\x00\x00\x00\x00\x00',
-    ],
-    (Ecu.eps, 0x7a1, None): [
-      b'8965B42170\x00\x00\x00\x00\x00\x00',
-      b'8965B42171\x00\x00\x00\x00\x00\x00',
-      b'8965B42180\x00\x00\x00\x00\x00\x00',
-      b'8965B42181\x00\x00\x00\x00\x00\x00',
-      b'\x028965B0R01200\x00\x00\x00\x008965B0R02200\x00\x00\x00\x00',
-      b'\x028965B0R01300\x00\x00\x00\x008965B0R02300\x00\x00\x00\x00',
-      b'\x028965B0R01400\x00\x00\x00\x008965B0R02400\x00\x00\x00\x00',
-    ],
-    (Ecu.fwdRadar, 0x750, 0xf): [
-      b'\x018821F3301100\x00\x00\x00\x00',
-      b'\x018821F3301200\x00\x00\x00\x00',
-      b'\x018821F3301300\x00\x00\x00\x00',
-      b'\x018821F3301400\x00\x00\x00\x00',
-    ],
-    (Ecu.fwdCamera, 0x750, 0x6d): [
-      b'\x028646F4203200\x00\x00\x00\x008646G26011A0\x00\x00\x00\x00',
-      b'\x028646F4203300\x00\x00\x00\x008646G26011A0\x00\x00\x00\x00',
-      b'\x028646F4203400\x00\x00\x00\x008646G2601200\x00\x00\x00\x00',
-      b'\x028646F4203500\x00\x00\x00\x008646G2601200\x00\x00\x00\x00',
-      b'\x028646F4203700\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
-      b'\x028646F4203800\x00\x00\x00\x008646G2601500\x00\x00\x00\x00',
-    ],
-  },
   CAR.RAV4H_TSS2_2022: {
     (Ecu.abs, 0x7b0, None): [
       b'\x01F15264283100\x00\x00\x00\x00',
@@ -1838,37 +1787,6 @@ FW_VERSIONS = {
       b'\x018966333T5100\x00\x00\x00\x00',
       b'\x018966333X6000\x00\x00\x00\x00',
       b'\x01896633T07000\x00\x00\x00\x00',
-    ],
-    (Ecu.abs, 0x7b0, None): [
-      b'\x01F152606281\x00\x00\x00\x00\x00\x00',
-      b'\x01F152606340\x00\x00\x00\x00\x00\x00',
-      b'\x01F152606461\x00\x00\x00\x00\x00\x00',
-      b'\x01F15260E031\x00\x00\x00\x00\x00\x00',
-    ],
-    (Ecu.eps, 0x7a1, None): [
-      b'8965B33252\x00\x00\x00\x00\x00\x00',
-      b'8965B33590\x00\x00\x00\x00\x00\x00',
-      b'8965B33690\x00\x00\x00\x00\x00\x00',
-      b'8965B33721\x00\x00\x00\x00\x00\x00',
-      b'8965B48271\x00\x00\x00\x00\x00\x00',
-    ],
-    (Ecu.fwdRadar, 0x750, 0xf): [
-      b'\x018821F3301100\x00\x00\x00\x00',
-      b'\x018821F3301200\x00\x00\x00\x00',
-      b'\x018821F3301400\x00\x00\x00\x00',
-      b'\x018821F6201300\x00\x00\x00\x00',
-    ],
-    (Ecu.fwdCamera, 0x750, 0x6d): [
-      b'\x028646F33030D0\x00\x00\x00\x008646G26011A0\x00\x00\x00\x00',
-      b'\x028646F3303200\x00\x00\x00\x008646G26011A0\x00\x00\x00\x00',
-      b'\x028646F3304100\x00\x00\x00\x008646G2601200\x00\x00\x00\x00',
-      b'\x028646F3304300\x00\x00\x00\x008646G2601500\x00\x00\x00\x00',
-      b'\x028646F3309100\x00\x00\x00\x008646G3304000\x00\x00\x00\x00',
-      b'\x028646F4810200\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
-    ],
-  },
-  CAR.LEXUS_ESH_TSS2: {
-    (Ecu.engine, 0x700, None): [
       b'\x028966333S8000\x00\x00\x00\x00897CF3302002\x00\x00\x00\x00',
       b'\x028966333S8000\x00\x00\x00\x00897CF3305001\x00\x00\x00\x00',
       b'\x028966333T0100\x00\x00\x00\x00897CF3305001\x00\x00\x00\x00',
@@ -1879,6 +1797,11 @@ FW_VERSIONS = {
       b'\x01896633T58000\x00\x00\x00\x00',
     ],
     (Ecu.abs, 0x7b0, None): [
+      b'\x01F152606281\x00\x00\x00\x00\x00\x00',
+      b'\x01F152606340\x00\x00\x00\x00\x00\x00',
+      b'\x01F152606461\x00\x00\x00\x00\x00\x00',
+      b'\x01F15260E031\x00\x00\x00\x00\x00\x00',
+      b'\x01F15260646200\x00\x00\x00\x00',
       b'F152633423\x00\x00\x00\x00\x00\x00',
       b'F152633680\x00\x00\x00\x00\x00\x00',
       b'F152633681\x00\x00\x00\x00\x00\x00',
@@ -1890,23 +1813,26 @@ FW_VERSIONS = {
       b'8965B33590\x00\x00\x00\x00\x00\x00',
       b'8965B33690\x00\x00\x00\x00\x00\x00',
       b'8965B33721\x00\x00\x00\x00\x00\x00',
+      b'8965B48271\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.fwdRadar, 0x750, 0xf): [
       b'\x018821F3301100\x00\x00\x00\x00',
       b'\x018821F3301200\x00\x00\x00\x00',
-      b'\x018821F3301300\x00\x00\x00\x00',
       b'\x018821F3301400\x00\x00\x00\x00',
       b'\x018821F6201300\x00\x00\x00\x00',
+      b'\x018821F3301300\x00\x00\x00\x00',
+      b'\x018821F6201400\x00\x00\x00\x00',
     ],
     (Ecu.fwdCamera, 0x750, 0x6d): [
-      b'\x028646F0610000\x00\x00\x00\x008646G3304000\x00\x00\x00\x00',
       b'\x028646F33030D0\x00\x00\x00\x008646G26011A0\x00\x00\x00\x00',
-      b'\x028646F3303100\x00\x00\x00\x008646G26011A0\x00\x00\x00\x00',
       b'\x028646F3303200\x00\x00\x00\x008646G26011A0\x00\x00\x00\x00',
       b'\x028646F3304100\x00\x00\x00\x008646G2601200\x00\x00\x00\x00',
-      b'\x028646F3304200\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
       b'\x028646F3304300\x00\x00\x00\x008646G2601500\x00\x00\x00\x00',
       b'\x028646F3309100\x00\x00\x00\x008646G3304000\x00\x00\x00\x00',
+      b'\x028646F4810200\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
+      b'\x028646F0610000\x00\x00\x00\x008646G3304000\x00\x00\x00\x00',
+      b'\x028646F3303100\x00\x00\x00\x008646G26011A0\x00\x00\x00\x00',
+      b'\x028646F3304200\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
     ],
   },
   CAR.LEXUS_ES: {
@@ -1959,17 +1885,30 @@ FW_VERSIONS = {
       b'\x01896637854000\x00\x00\x00\x00',
       b'\x01896637878000\x00\x00\x00\x00',
     ],
+    (Ecu.engine, 0x7e0, None): [
+      b'\x0237841000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00',
+      b'\x0237842000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00',
+      b'\x0237880000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00',
+      b'\x0237882000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00',
+      b'\x0237886000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00',
+    ],
     (Ecu.abs, 0x7b0, None): [
       b'F152678130\x00\x00\x00\x00\x00\x00',
       b'F152678140\x00\x00\x00\x00\x00\x00',
+      b'F152678160\x00\x00\x00\x00\x00\x00',
+      b'F152678170\x00\x00\x00\x00\x00\x00',
+      b'F152678171\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.dsu, 0x791, None): [
       b'881517803100\x00\x00\x00\x00',
       b'881517803300\x00\x00\x00\x00',
+      b'881517804300\x00\x00\x00\x00',
+      b'881517804100\x00\x00\x00\x00',
     ],
     (Ecu.eps, 0x7a1, None): [
       b'8965B78060\x00\x00\x00\x00\x00\x00',
       b'8965B78080\x00\x00\x00\x00\x00\x00',
+      b'8965B78100\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.fwdRadar, 0x750, 0xf): [
       b'8821F4702100\x00\x00\x00\x00',
@@ -1990,29 +1929,13 @@ FW_VERSIONS = {
       b'\x018966378B2000\x00\x00\x00\x00',
       b'\x018966378B3100\x00\x00\x00\x00',
     ],
-    (Ecu.abs, 0x7b0, None): [
-      b'\x01F152678221\x00\x00\x00\x00\x00\x00',
-    ],
-    (Ecu.eps, 0x7a1, None): [
-      b'8965B78120\x00\x00\x00\x00\x00\x00',
-    ],
-    (Ecu.fwdRadar, 0x750, 0xf): [
-      b"\x018821F3301400\x00\x00\x00\x00",
-      b'\x018821F3301200\x00\x00\x00\x00',
-      b'\x018821F3301300\x00\x00\x00\x00',
-    ],
-    (Ecu.fwdCamera, 0x750, 0x6d): [
-      b'\x028646F78030A0\x00\x00\x00\x008646G2601200\x00\x00\x00\x00',
-      b'\x028646F7803100\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
-    ],
-  },
-  CAR.LEXUS_NXH_TSS2: {
     (Ecu.engine, 0x7e0, None): [
       b'\x0237887000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00',
       b'\x02378A0000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00',
       b'\x02378F4000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.abs, 0x7b0, None): [
+      b'\x01F152678221\x00\x00\x00\x00\x00\x00',
       b'F152678210\x00\x00\x00\x00\x00\x00',
       b'F152678211\x00\x00\x00\x00\x00\x00',
     ],
@@ -2020,7 +1943,8 @@ FW_VERSIONS = {
       b'8965B78120\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.fwdRadar, 0x750, 0xf): [
-      b'\x018821F3301400\x00\x00\x00\x00',
+      b"\x018821F3301400\x00\x00\x00\x00",
+      b'\x018821F3301200\x00\x00\x00\x00',
       b'\x018821F3301300\x00\x00\x00\x00',
     ],
     (Ecu.fwdCamera, 0x750, 0x6d): [
@@ -2028,37 +1952,6 @@ FW_VERSIONS = {
       b'\x028646F7803100\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
     ],
   },
-  CAR.LEXUS_NXH: {
-    (Ecu.engine, 0x7e0, None): [
-      b'\x0237841000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00',
-      b'\x0237842000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00',
-      b'\x0237880000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00',
-      b'\x0237882000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00',
-      b'\x0237886000\x00\x00\x00\x00\x00\x00\x00\x00A4701000\x00\x00\x00\x00\x00\x00\x00\x00',
-    ],
-    (Ecu.abs, 0x7b0, None): [
-      b'F152678160\x00\x00\x00\x00\x00\x00',
-      b'F152678170\x00\x00\x00\x00\x00\x00',
-      b'F152678171\x00\x00\x00\x00\x00\x00',
-    ],
-    (Ecu.dsu, 0x791, None): [
-      b'881517804300\x00\x00\x00\x00',
-      b'881517804100\x00\x00\x00\x00',
-    ],
-    (Ecu.eps, 0x7a1, None): [
-      b'8965B78060\x00\x00\x00\x00\x00\x00',
-      b'8965B78080\x00\x00\x00\x00\x00\x00',
-      b'8965B78100\x00\x00\x00\x00\x00\x00',
-    ],
-    (Ecu.fwdRadar, 0x750, 0xf): [
-      b'8821F4702300\x00\x00\x00\x00',
-      b'8821F4702100\x00\x00\x00\x00',
-    ],
-    (Ecu.fwdCamera, 0x750, 0x6d): [
-      b'8646F7801300\x00\x00\x00\x00',
-      b'8646F7801100\x00\x00\x00\x00',
-    ],
-  },
   CAR.LEXUS_RC: {
     (Ecu.engine, 0x700, None): [
       b'\x01896632478200\x00\x00\x00\x00',
@@ -2204,29 +2097,6 @@ FW_VERSIONS = {
       b'\x018966348X0000\x00\x00\x00\x00',
       b'\x01896630ED5000\x00\x00\x00\x00',
     ],
-    (Ecu.abs, 0x7b0, None): [
-      b'\x01F15260E031\x00\x00\x00\x00\x00\x00',
-      b'\x01F15260E041\x00\x00\x00\x00\x00\x00',
-      b'\x01F152648781\x00\x00\x00\x00\x00\x00',
-      b'\x01F152648801\x00\x00\x00\x00\x00\x00',
-    ],
-    (Ecu.eps, 0x7a1, None): [
-      b'8965B48261\x00\x00\x00\x00\x00\x00',
-      b'8965B48271\x00\x00\x00\x00\x00\x00',
-    ],
-    (Ecu.fwdRadar, 0x750, 0xf): [
-      b'\x018821F3301100\x00\x00\x00\x00',
-      b'\x018821F3301300\x00\x00\x00\x00',
-      b'\x018821F3301400\x00\x00\x00\x00',
-    ],
-    (Ecu.fwdCamera, 0x750, 0x6d): [
-      b'\x028646F4810100\x00\x00\x00\x008646G2601200\x00\x00\x00\x00',
-      b'\x028646F4810200\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
-      b'\x028646F4810300\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
-      b'\x028646F4810400\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
-    ],
-  },
-  CAR.LEXUS_RXH_TSS2: {
     (Ecu.engine, 0x7e0, None): [
       b'\x02348X4000\x00\x00\x00\x00\x00\x00\x00\x00A4802000\x00\x00\x00\x00\x00\x00\x00\x00',
       b'\x02348X5000\x00\x00\x00\x00\x00\x00\x00\x00A4802000\x00\x00\x00\x00\x00\x00\x00\x00',
@@ -2234,25 +2104,34 @@ FW_VERSIONS = {
       b'\x02348Y3000\x00\x00\x00\x00\x00\x00\x00\x00A4802000\x00\x00\x00\x00\x00\x00\x00\x00',
       b'\x0234D14000\x00\x00\x00\x00\x00\x00\x00\x00A4802000\x00\x00\x00\x00\x00\x00\x00\x00',
       b'\x0234D16000\x00\x00\x00\x00\x00\x00\x00\x00A4802000\x00\x00\x00\x00\x00\x00\x00\x00',
+      b'\x02348U2000\x00\x00\x00\x00\x00\x00\x00\x00A4802000\x00\x00\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.abs, 0x7b0, None): [
+      b'\x01F15260E031\x00\x00\x00\x00\x00\x00',
+      b'\x01F15260E041\x00\x00\x00\x00\x00\x00',
+      b'\x01F152648781\x00\x00\x00\x00\x00\x00',
+      b'\x01F152648801\x00\x00\x00\x00\x00\x00',
       b'F152648831\x00\x00\x00\x00\x00\x00',
       b'F152648891\x00\x00\x00\x00\x00\x00',
       b'F152648D00\x00\x00\x00\x00\x00\x00',
       b'F152648D60\x00\x00\x00\x00\x00\x00',
       b'F152648811\x00\x00\x00\x00\x00\x00',
+      b'F152648C80\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.eps, 0x7a1, None): [
       b'8965B48261\x00\x00\x00\x00\x00\x00',
       b'8965B48271\x00\x00\x00\x00\x00\x00',
     ],
     (Ecu.fwdRadar, 0x750, 0xf): [
+      b'\x018821F3301100\x00\x00\x00\x00',
+      b'\x018821F3301300\x00\x00\x00\x00',
       b'\x018821F3301400\x00\x00\x00\x00',
     ],
     (Ecu.fwdCamera, 0x750, 0x6d): [
       b'\x028646F4810100\x00\x00\x00\x008646G2601200\x00\x00\x00\x00',
       b'\x028646F4810200\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
       b'\x028646F4810300\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
+      b'\x028646F4810400\x00\x00\x00\x008646G2601400\x00\x00\x00\x00',
     ],
   },
   CAR.PRIUS_TSS2: {
@@ -2345,10 +2224,8 @@ DBC = {
   CAR.LEXUS_RX: dbc_dict('toyota_tnga_k_pt_generated', 'toyota_adas'),
   CAR.LEXUS_RXH: dbc_dict('toyota_tnga_k_pt_generated', 'toyota_adas'),
   CAR.LEXUS_RX_TSS2: dbc_dict('toyota_nodsu_pt_generated', 'toyota_tss2_adas'),
-  CAR.LEXUS_RXH_TSS2: dbc_dict('toyota_nodsu_pt_generated', 'toyota_tss2_adas'),
   CAR.CHR: dbc_dict('toyota_nodsu_pt_generated', 'toyota_adas'),
   CAR.CHR_TSS2: dbc_dict('toyota_nodsu_pt_generated', None),
-  CAR.CHRH: dbc_dict('toyota_nodsu_pt_generated', 'toyota_adas'),
   CAR.CHRH_TSS2: dbc_dict('toyota_nodsu_pt_generated', None),
   CAR.CAMRY: dbc_dict('toyota_nodsu_pt_generated', 'toyota_adas'),
   CAR.CAMRYH: dbc_dict('toyota_nodsu_pt_generated', 'toyota_adas'),
@@ -2367,22 +2244,17 @@ DBC = {
   CAR.RAV4_TSS2_2022: dbc_dict('toyota_nodsu_pt_generated', None),
   CAR.RAV4_TSS2_2023: dbc_dict('toyota_nodsu_pt_generated', None),
   CAR.COROLLA_TSS2: dbc_dict('toyota_nodsu_pt_generated', 'toyota_tss2_adas'),
-  CAR.COROLLAH_TSS2: dbc_dict('toyota_nodsu_pt_generated', 'toyota_tss2_adas'),
   CAR.LEXUS_ES: dbc_dict('toyota_new_mc_pt_generated', 'toyota_adas'),
   CAR.LEXUS_ES_TSS2: dbc_dict('toyota_nodsu_pt_generated', 'toyota_tss2_adas'),
-  CAR.LEXUS_ESH_TSS2: dbc_dict('toyota_nodsu_pt_generated', 'toyota_tss2_adas'),
   CAR.LEXUS_ESH: dbc_dict('toyota_new_mc_pt_generated', 'toyota_adas'),
   CAR.SIENNA: dbc_dict('toyota_tnga_k_pt_generated', 'toyota_adas'),
   CAR.LEXUS_IS: dbc_dict('toyota_tnga_k_pt_generated', 'toyota_adas'),
   CAR.LEXUS_IS_TSS2: dbc_dict('toyota_nodsu_pt_generated', 'toyota_tss2_adas'),
   CAR.LEXUS_CTH: dbc_dict('toyota_new_mc_pt_generated', 'toyota_adas'),
-  CAR.RAV4H_TSS2: dbc_dict('toyota_nodsu_pt_generated', 'toyota_tss2_adas'),
   CAR.RAV4H_TSS2_2022: dbc_dict('toyota_nodsu_pt_generated', None),
   CAR.RAV4H_TSS2_2023: dbc_dict('toyota_nodsu_pt_generated', None),
-  CAR.LEXUS_NXH: dbc_dict('toyota_tnga_k_pt_generated', 'toyota_adas'),
   CAR.LEXUS_NX: dbc_dict('toyota_tnga_k_pt_generated', 'toyota_adas'),
   CAR.LEXUS_NX_TSS2: dbc_dict('toyota_nodsu_pt_generated', 'toyota_tss2_adas'),
-  CAR.LEXUS_NXH_TSS2: dbc_dict('toyota_nodsu_pt_generated', 'toyota_tss2_adas'),
   CAR.PRIUS_TSS2: dbc_dict('toyota_nodsu_pt_generated', 'toyota_tss2_adas'),
   CAR.MIRAI: dbc_dict('toyota_nodsu_pt_generated', 'toyota_tss2_adas'),
   CAR.ALPHARD_TSS2: dbc_dict('toyota_nodsu_pt_generated', 'toyota_tss2_adas'),
@@ -2393,12 +2265,12 @@ DBC = {
 EPS_SCALE = defaultdict(lambda: 73, {CAR.PRIUS: 66, CAR.COROLLA: 88, CAR.LEXUS_IS: 77, CAR.LEXUS_RC: 77, CAR.LEXUS_CTH: 100, CAR.PRIUS_V: 100})
 
 # Toyota/Lexus Safety Sense 2.0 and 2.5
-TSS2_CAR = {CAR.RAV4_TSS2, CAR.RAV4_TSS2_2022, CAR.RAV4_TSS2_2023, CAR.COROLLA_TSS2, CAR.COROLLAH_TSS2, CAR.LEXUS_ES_TSS2, CAR.LEXUS_ESH_TSS2,
+TSS2_CAR = {CAR.RAV4_TSS2, CAR.RAV4_TSS2_2022, CAR.RAV4_TSS2_2023, CAR.COROLLA_TSS2, CAR.LEXUS_ES_TSS2,
-            CAR.RAV4H_TSS2, CAR.RAV4H_TSS2_2022, CAR.RAV4H_TSS2_2023, CAR.LEXUS_RX_TSS2, CAR.LEXUS_RXH_TSS2, CAR.HIGHLANDER_TSS2,
+            CAR.RAV4H_TSS2_2022, CAR.RAV4H_TSS2_2023, CAR.LEXUS_RX_TSS2, CAR.HIGHLANDER_TSS2,
             CAR.HIGHLANDERH_TSS2, CAR.PRIUS_TSS2, CAR.CAMRY_TSS2, CAR.CAMRYH_TSS2, CAR.LEXUS_IS_TSS2, CAR.MIRAI, CAR.LEXUS_NX_TSS2,
-            CAR.LEXUS_NXH_TSS2, CAR.ALPHARD_TSS2, CAR.AVALON_TSS2, CAR.AVALONH_TSS2, CAR.ALPHARDH_TSS2, CAR.CHR_TSS2, CAR.CHRH_TSS2}
+            CAR.ALPHARD_TSS2, CAR.AVALON_TSS2, CAR.AVALONH_TSS2, CAR.ALPHARDH_TSS2, CAR.CHR_TSS2, CAR.CHRH_TSS2}
 
-NO_DSU_CAR = TSS2_CAR | {CAR.CHR, CAR.CHRH, CAR.CAMRY, CAR.CAMRYH}
+NO_DSU_CAR = TSS2_CAR | {CAR.CHR, CAR.CAMRY, CAR.CAMRYH}
 
 # the DSU uses the AEB message for longitudinal on these cars
 UNSUPPORTED_DSU_CAR = {CAR.LEXUS_IS, CAR.LEXUS_RC}
@@ -2409,10 +2281,5 @@ RADAR_ACC_CAR = {CAR.RAV4H_TSS2_2022, CAR.RAV4_TSS2_2022, CAR.RAV4H_TSS2_2023, C
 # these cars use the Lane Tracing Assist (LTA) message for lateral control
 ANGLE_CONTROL_CAR = {CAR.RAV4H_TSS2_2023, CAR.RAV4_TSS2_2023}
 
-EV_HYBRID_CAR = {CAR.AVALONH_2019, CAR.AVALONH_TSS2, CAR.CAMRYH, CAR.CAMRYH_TSS2, CAR.CHRH, CAR.CHRH_TSS2, CAR.COROLLAH_TSS2,
-                 CAR.HIGHLANDERH, CAR.HIGHLANDERH_TSS2, CAR.PRIUS, CAR.PRIUS_V, CAR.RAV4H, CAR.RAV4H_TSS2, CAR.RAV4H_TSS2_2022,
-                 CAR.RAV4H_TSS2_2023, CAR.LEXUS_CTH, CAR.MIRAI, CAR.LEXUS_ESH, CAR.LEXUS_ESH_TSS2, CAR.LEXUS_NXH, CAR.LEXUS_RXH,
-                 CAR.LEXUS_RXH_TSS2, CAR.LEXUS_NXH_TSS2, CAR.PRIUS_TSS2, CAR.ALPHARDH_TSS2}
-
 # no resume button press required
 NO_STOP_TIMER_CAR = TSS2_CAR | {CAR.PRIUS_V, CAR.RAV4H, CAR.HIGHLANDERH, CAR.HIGHLANDER, CAR.SIENNA, CAR.LEXUS_ESH}

selfdrive/car/volkswagen/values.py

@@ -245,7 +245,7 @@ CAR_INFO: Dict[str, Union[VWCarInfo, List[VWCarInfo]]] = {
   CAR.TOURAN_MK2: VWCarInfo("Volkswagen Touran 2016-23"),
   CAR.TRANSPORTER_T61: [
     VWCarInfo("Volkswagen Caravelle 2020"),
-    VWCarInfo("Volkswagen California 2021"),
+    VWCarInfo("Volkswagen California 2021-23"),
   ],
   CAR.TROC_MK1: VWCarInfo("Volkswagen T-Roc 2021", footnotes=[Footnote.VW_MQB_A0]),
   CAR.AUDI_A3_MK3: [
@@ -774,15 +774,18 @@ FW_VERSIONS = {
     (Ecu.engine, 0x7e0, None): [
       b'\xf1\x8704E906027NJ\xf1\x891445',
       b'\xf1\x8704E906027NP\xf1\x891286',
+      b'\xf1\x8705E906013BD\xf1\x892496',
       b'\xf1\x8705E906013E \xf1\x891624',
     ],
     (Ecu.transmission, 0x7e1, None): [
       b'\xf1\x8709G927158EM\xf1\x893812',
       b'\xf1\x8709S927158BL\xf1\x893791',
+      b'\xf1\x8709S927158DN\xf1\x893946',
       b'\xf1\x8709S927158FF\xf1\x893876',
     ],
     (Ecu.srs, 0x715, None): [
       b'\xf1\x875Q0959655CB\xf1\x890421\xf1\x82\x1311111111333500314646021450149333613100',
+      b'\xf1\x875Q0959655CB\xf1\x890421\xf1\x82\x1312111111333500314646021550159333613100',
       b'\xf1\x875Q0959655CE\xf1\x890421\xf1\x82\x1311110011333300314240021350139333613100',
     ],
     (Ecu.eps, 0x712, None): [
@@ -916,12 +919,14 @@ FW_VERSIONS = {
       b'\xf1\x8704L906057AP\xf1\x891186',
       b'\xf1\x8704L906057N \xf1\x890413',
       b'\xf1\x8705L906023E \xf1\x891352',
+      b'\xf1\x8705L906023MR\xf1\x892582',
     ],
     (Ecu.transmission, 0x7e1, None): [
       b'\xf1\x870BT300012G \xf1\x893102',
       b'\xf1\x870BT300012E \xf1\x893105',
       b'\xf1\x870BT300046R \xf1\x893102',
       b'\xf1\x870DV300012B \xf1\x893701',
+      b'\xf1\x870DV300012B \xf1\x893702',
     ],
     (Ecu.srs, 0x715, None): [
       b'\xf1\x872Q0959655AE\xf1\x890506\xf1\x82\x1316170411110411--04041704161611152S1411',
@@ -1063,10 +1068,12 @@ FW_VERSIONS = {
       b'\xf1\x8705L906022M \xf1\x890901',
       b'\xf1\x8783A906259  \xf1\x890001',
       b'\xf1\x8783A906259  \xf1\x890005',
+      b'\xf1\x8783A906259C \xf1\x890002',
       b'\xf1\x8783A906259F \xf1\x890001',
     ],
     (Ecu.transmission, 0x7e1, None): [
       b'\xf1\x8709G927158CN\xf1\x893608',
+      b'\xf1\x8709G927158FL\xf1\x893758',
       b'\xf1\x8709G927158GP\xf1\x893937',
       b'\xf1\x870GC300045D \xf1\x892802',
       b'\xf1\x870GC300046F \xf1\x892701',
@@ -1081,6 +1088,7 @@ FW_VERSIONS = {
       b'\xf1\x875Q0910143C \xf1\x892211\xf1\x82\x0567G6000300',
       b'\xf1\x875Q0910143C \xf1\x892211\xf1\x82\x0567G6000800',
       b'\xf1\x875QF909144B \xf1\x895582\xf1\x82\x0571G60533A1',
+      b'\xf1\x875QF909144B \xf1\x895582\xf1\x82\x0571G60733A1',
       b'\xf1\x875TA907145D \xf1\x891051\xf1\x82\x001PG60A1P7N',
     ],
     (Ecu.fwdRadar, 0x757, None): [

selfdrive/hardware

@@ -1 +0,0 @@
-../system/hardware/

selfdrive/modeld/modeld.py

@@ -121,7 +121,7 @@ def main():
 
   while not vipc_client_main.connect(False):
     time.sleep(0.1)
-  while not vipc_client_extra.connect(False):
+  while use_extra_client and not vipc_client_extra.connect(False):
     time.sleep(0.1)
 
   cloudlog.warning(f"connected main cam with buffer size: {vipc_client_main.buffer_len} ({vipc_client_main.width} x {vipc_client_main.height})")

selfdrive/rtshield.py

@@ -1,21 +0,0 @@
-#!/usr/bin/env python3
-import os
-import time
-from typing import NoReturn
-
-from openpilot.common.realtime import set_core_affinity, set_realtime_priority
-
-# RT shield - ensure CPU 3 always remains available for RT processes
-#   runs as SCHED_FIFO with minimum priority to ensure kthreads don't
-#   get scheduled onto CPU 3, but it's always preemptible by realtime
-#   openpilot processes
-
-def main() -> NoReturn:
-  set_core_affinity([int(os.getenv("CORE", "3")), ])
-  set_realtime_priority(1)
-
-  while True:
-    time.sleep(0.000001)
-
-if __name__ == "__main__":
-  main()

selfdrive/sentry.py

@@ -42,11 +42,11 @@ def set_tag(key: str, value: str) -> None:
   sentry_sdk.set_tag(key, value)
 
 
-def init(project: SentryProject) -> None:
+def init(project: SentryProject) -> bool:
   # forks like to mess with this, so double check
   comma_remote = is_comma_remote() and "commaai" in get_origin(default="")
   if not comma_remote or not is_registered_device() or PC:
-    return
+    return False
 
   env = "release" if is_tested_branch() else "master"
   dongle_id = Params().get("DongleId", encoding='utf-8')
@@ -73,3 +73,5 @@ def init(project: SentryProject) -> None:
 
   if project == SentryProject.SELFDRIVE:
     sentry_sdk.Hub.current.start_session()
+
+  return True

selfdrive/test/process_replay/compare_logs.py

@@ -5,6 +5,7 @@ import capnp
 import numbers
 import dictdiffer
 from collections import Counter
+from typing import Dict
 
 from openpilot.tools.lib.logreader import LogReader
 
@@ -89,7 +90,52 @@ def compare_logs(log1, log2, ignore_fields=None, ignore_msgs=None, tolerance=Non
   return diff
 
 
+def format_diff(results, log_paths, ref_commit):
+  diff1, diff2 = "", ""
+  diff2 += f"***** tested against commit {ref_commit} *****\n"
+
+  failed = False
+  for segment, result in list(results.items()):
+    diff1 += f"***** results for segment {segment} *****\n"
+    diff2 += f"***** differences for segment {segment} *****\n"
+
+    for proc, diff in list(result.items()):
+      # long diff
+      diff2 += f"*** process: {proc} ***\n"
+      diff2 += f"\tref: {log_paths[segment][proc]['ref']}\n"
+      diff2 += f"\tnew: {log_paths[segment][proc]['new']}\n\n"
+
+      # short diff
+      diff1 += f"    {proc}\n"
+      if isinstance(diff, str):
+        diff1 += f"        ref: {log_paths[segment][proc]['ref']}\n"
+        diff1 += f"        new: {log_paths[segment][proc]['new']}\n\n"
+        diff1 += f"        {diff}\n"
+        failed = True
+      elif len(diff):
+        diff1 += f"        ref: {log_paths[segment][proc]['ref']}\n"
+        diff1 += f"        new: {log_paths[segment][proc]['new']}\n\n"
+
+        cnt: Dict[str, int] = {}
+        for d in diff:
+          diff2 += f"\t{str(d)}\n"
+
+          k = str(d[1])
+          cnt[k] = 1 if k not in cnt else cnt[k] + 1
+
+        for k, v in sorted(cnt.items()):
+          diff1 += f"        {k}: {v}\n"
+        failed = True
+  return diff1, diff2, failed
+
+
 if __name__ == "__main__":
   log1 = list(LogReader(sys.argv[1]))
   log2 = list(LogReader(sys.argv[2]))
-  print(compare_logs(log1, log2, sys.argv[3:]))
+  ignore_fields = sys.argv[3:] or ["logMonoTime", "controlsState.startMonoTime", "controlsState.cumLagMs"]
+  results = {"segment": {"proc": compare_logs(log1, log2, ignore_fields)}}
+  log_paths = {"segment": {"proc": {"ref": sys.argv[1], "new": sys.argv[2]}}}
+  diff1, diff2, failed = format_diff(results, log_paths, None)
+
+  print(diff2)
+  print(diff1)

selfdrive/test/process_replay/migration.py

@@ -2,17 +2,95 @@ from collections import defaultdict
 
 from cereal import messaging
 from openpilot.selfdrive.test.process_replay.vision_meta import meta_from_encode_index
+from openpilot.selfdrive.car.toyota.values import EPS_SCALE
+from openpilot.selfdrive.manager.process_config import managed_processes
+from panda import Panda
 
 
-def migrate_all(lr, old_logtime=False, camera_states=False):
+def migrate_all(lr, old_logtime=False, manager_states=False, panda_states=False, camera_states=False):
   msgs = migrate_sensorEvents(lr, old_logtime)
   msgs = migrate_carParams(msgs, old_logtime)
+  if manager_states:
+    msgs = migrate_managerState(msgs)
+  if panda_states:
+    msgs = migrate_pandaStates(msgs)
+    msgs = migrate_peripheralState(msgs)
   if camera_states:
     msgs = migrate_cameraStates(msgs)
 
   return msgs
 
 
+def migrate_managerState(lr):
+  all_msgs = []
+  for msg in lr:
+    if msg.which() != "managerState":
+      all_msgs.append(msg)
+      continue
+
+    new_msg = msg.as_builder()
+    new_msg.managerState.processes = [{'name': name, 'running': True} for name in managed_processes]
+    all_msgs.append(new_msg.as_reader())
+
+  return all_msgs
+
+
+def migrate_pandaStates(lr):
+  all_msgs = []
+  # TODO: safety param migration should be handled automatically
+  safety_param_migration = {
+    "TOYOTA PRIUS 2017": EPS_SCALE["TOYOTA PRIUS 2017"] | Panda.FLAG_TOYOTA_STOCK_LONGITUDINAL,
+    "TOYOTA RAV4 2017": EPS_SCALE["TOYOTA RAV4 2017"] | Panda.FLAG_TOYOTA_ALT_BRAKE,
+    "KIA EV6 2022": Panda.FLAG_HYUNDAI_EV_GAS | Panda.FLAG_HYUNDAI_CANFD_HDA2,
+  }
+
+  # Migrate safety param base on carState
+  CP = next((m.carParams for m in lr if m.which() == 'carParams'), None)
+  assert CP is not None, "carParams message not found"
+  if CP.carFingerprint in safety_param_migration:
+    safety_param = safety_param_migration[CP.carFingerprint]
+  elif len(CP.safetyConfigs):
+    safety_param = CP.safetyConfigs[0].safetyParam
+    if CP.safetyConfigs[0].safetyParamDEPRECATED != 0:
+      safety_param = CP.safetyConfigs[0].safetyParamDEPRECATED
+  else:
+    safety_param = CP.safetyParamDEPRECATED
+
+  for msg in lr:
+    if msg.which() == 'pandaStateDEPRECATED':
+      new_msg = messaging.new_message('pandaStates', 1)
+      new_msg.valid = msg.valid
+      new_msg.logMonoTime = msg.logMonoTime
+      new_msg.pandaStates[0] = msg.pandaStateDEPRECATED
+      new_msg.pandaStates[0].safetyParam = safety_param
+      all_msgs.append(new_msg.as_reader())
+    elif msg.which() == 'pandaStates':
+      new_msg = msg.as_builder()
+      new_msg.pandaStates[-1].safetyParam = safety_param
+      all_msgs.append(new_msg.as_reader())
+    else:
+      all_msgs.append(msg)
+
+  return all_msgs
+
+
+def migrate_peripheralState(lr):
+  if any(msg.which() == "peripheralState" for msg in lr):
+    return lr
+
+  all_msg = []
+  for msg in lr:
+    all_msg.append(msg)
+    if msg.which() not in ["pandaStates", "pandaStateDEPRECATED"]:
+      continue
+
+    new_msg = messaging.new_message("peripheralState")
+    new_msg.logMonoTime = msg.logMonoTime
+    all_msg.append(new_msg.as_reader())
+
+  return all_msg
+
+
 def migrate_cameraStates(lr):
   all_msgs = []
   frame_to_encode_id = defaultdict(dict)
@@ -42,6 +120,10 @@ def migrate_cameraStates(lr):
     new_camera_state = getattr(new_msg, new_msg.which())
     new_camera_state.frameId = encode_id
     new_camera_state.encodeId = encode_id
+    # timestampSof was added later so it might be missing on some old segments
+    if camera_state.timestampSof == 0 and camera_state.timestampEof > 25000000:
+      new_camera_state.timestampSof = camera_state.timestampEof - 18000000
+    else:
       new_camera_state.timestampSof = camera_state.timestampSof
     new_camera_state.timestampEof = camera_state.timestampEof
     new_msg.logMonoTime = msg.logMonoTime

selfdrive/test/process_replay/model_replay.py

@@ -10,8 +10,7 @@ from openpilot.common.params import Params
 from openpilot.system.hardware import PC
 from openpilot.selfdrive.manager.process_config import managed_processes
 from openpilot.selfdrive.test.openpilotci import BASE_URL, get_url
-from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs
+from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs, format_diff
-from openpilot.selfdrive.test.process_replay.test_processes import format_diff
 from openpilot.selfdrive.test.process_replay.process_replay import get_process_config, replay_process
 from openpilot.system.version import get_commit
 from openpilot.tools.lib.framereader import FrameReader

selfdrive/test/process_replay/model_replay_ref_commit

@@ -1 +1 @@
-3bb23e270a3a7219cfeedadd8d085c32fc571c0d
+ed2d58ec217fafb7b6b8f5e27ec622acd9e734f4

selfdrive/test/process_replay/process_replay.py

@@ -204,7 +204,7 @@ class ProcessContainer:
   def _setup_vision_ipc(self, all_msgs):
     assert len(self.cfg.vision_pubs) != 0
 
-    device_type = next(msg.initData.deviceType for msg in all_msgs if msg.which() == "initData")
+    device_type = next(str(msg.initData.deviceType) for msg in all_msgs if msg.which() == "initData")
 
     vipc_server = VisionIpcServer("camerad")
     streams_metas = available_streams(all_msgs)
@@ -214,6 +214,7 @@ class ProcessContainer:
     vipc_server.start_listener()
 
     self.vipc_server = vipc_server
+    self.cfg.vision_pubs = [meta.camera_state for meta in streams_metas if meta.camera_state in self.cfg.vision_pubs]
 
   def _start_process(self):
     if self.capture is not None:
@@ -393,9 +394,8 @@ class ModeldCameraSyncRcvCallback:
     self.is_dual_camera = True
 
   def __call__(self, msg, cfg, frame):
-    if msg.which() == "initData":
+    self.is_dual_camera = len(cfg.vision_pubs) == 2
-      self.is_dual_camera = msg.initData.deviceType in ["tici", "tizi"]
+    if msg.which() == "roadCameraState":
-    elif msg.which() == "roadCameraState":
       self.road_present = True
     elif msg.which() == "wideRoadCameraState":
       self.wide_road_present = True
@@ -653,7 +653,10 @@ def replay_process(
   else:
     cfgs = [cfg]
 
-  all_msgs = migrate_all(lr, old_logtime=True, camera_states=any(len(cfg.vision_pubs) != 0 for cfg in cfgs))
+  all_msgs = migrate_all(lr, old_logtime=True,
+                         manager_states=True,
+                         panda_states=any("pandaStates" in cfg.pubs for cfg in cfgs),
+                         camera_states=any(len(cfg.vision_pubs) != 0 for cfg in cfgs))
   process_logs = _replay_multi_process(cfgs, all_msgs, frs, fingerprint, custom_params, captured_output_store, disable_progress)
 
   if return_all_logs:
@@ -681,14 +684,13 @@ def _replay_multi_process(
     env_config = generate_environ_config(CP=CP)
 
   # validate frs and vision pubs
-  for cfg in cfgs:
+  all_vision_pubs = [pub for cfg in cfgs for pub in cfg.vision_pubs]
-    if len(cfg.vision_pubs) == 0:
+  if len(all_vision_pubs) != 0:
-      continue
-
     assert frs is not None, "frs must be provided when replaying process using vision streams"
-    assert all(meta_from_camera_state(st) is not None for st in cfg.vision_pubs),\
+    assert all(meta_from_camera_state(st) is not None for st in all_vision_pubs), \
-                                                     f"undefined vision stream spotted, probably misconfigured process: {cfg.vision_pubs}"
+                                                          f"undefined vision stream spotted, probably misconfigured process: (vision pubs: {all_vision_pubs})"
-    assert all(st in frs for st in cfg.vision_pubs), f"frs for this process must contain following vision streams: {cfg.vision_pubs}"
+    required_vision_pubs = {m.camera_state for m in available_streams(lr)} & set(all_vision_pubs)
+    assert all(st in frs for st in required_vision_pubs), f"frs for this process must contain following vision streams: {required_vision_pubs}"
 
   all_msgs = sorted(lr, key=lambda msg: msg.logMonoTime)
   log_msgs = []

selfdrive/test/process_replay/ref_commit

@@ -1 +1 @@
-7553d9123f418fd81ef811df4e130ab153ac489a
+d05b67ec66630521e8cedb90981002d57d738f6d

selfdrive/test/process_replay/regen.py

@@ -6,7 +6,7 @@ import capnp
 
 from typing import Union, Iterable, Optional, List, Any, Dict, Tuple
 
-from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, FAKEDATA, replay_process, get_process_config, \
+from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, FAKEDATA, ProcessConfig, replay_process, get_process_config, \
                                                                    check_openpilot_enabled, get_custom_params_from_lr
 from openpilot.selfdrive.test.update_ci_routes import upload_route
 from openpilot.tools.lib.route import Route
@@ -17,62 +17,68 @@ from openpilot.tools.lib.helpers import save_log
 
 def regen_segment(
   lr: Union[LogReader, List[capnp._DynamicStructReader]], frs: Optional[Dict[str, Any]] = None,
-  daemons: Union[str, Iterable[str]] = "all", disable_tqdm: bool = False
+  processes: Iterable[ProcessConfig] = CONFIGS, disable_tqdm: bool = False
 ) -> List[capnp._DynamicStructReader]:
-  if not isinstance(daemons, str) and not hasattr(daemons, "__iter__"):
-    raise ValueError("whitelist_proc must be a string or iterable")
-
   all_msgs = sorted(lr, key=lambda m: m.logMonoTime)
   custom_params = get_custom_params_from_lr(all_msgs)
 
-  if daemons != "all":
+  print("Replayed processes:", [p.proc_name for p in processes])
-    if isinstance(daemons, str):
-      raise ValueError(f"Invalid value for daemons: {daemons}")
-
-    replayed_processes = []
-    for d in daemons:
-      cfg = get_process_config(d)
-      replayed_processes.append(cfg)
-  else:
-    replayed_processes = CONFIGS
-
-  print("Replayed processes:", [p.proc_name for p in replayed_processes])
   print("\n\n", "*"*30, "\n\n", sep="")
 
-  output_logs = replay_process(replayed_processes, all_msgs, frs, return_all_logs=True, custom_params=custom_params, disable_progress=disable_tqdm)
+  output_logs = replay_process(processes, all_msgs, frs, return_all_logs=True, custom_params=custom_params, disable_progress=disable_tqdm)
 
   return output_logs
 
 
-def setup_data_readers(route: str, sidx: int, use_route_meta: bool) -> Tuple[LogReader, Dict[str, Any]]:
+def setup_data_readers(
+    route: str, sidx: int, use_route_meta: bool, needs_driver_cam: bool = True, needs_road_cam: bool = True
+) -> Tuple[LogReader, Dict[str, Any]]:
   if use_route_meta:
     r = Route(route)
     lr = LogReader(r.log_paths()[sidx])
     frs = {}
-    if len(r.camera_paths()) > sidx and r.camera_paths()[sidx] is not None:
+    if needs_road_cam and len(r.camera_paths()) > sidx and r.camera_paths()[sidx] is not None:
       frs['roadCameraState'] = FrameReader(r.camera_paths()[sidx])
-    if len(r.ecamera_paths()) > sidx and r.ecamera_paths()[sidx] is not None:
+    if needs_road_cam and  len(r.ecamera_paths()) > sidx and r.ecamera_paths()[sidx] is not None:
-      frs['wideCameraState'] = FrameReader(r.ecamera_paths()[sidx])
+      frs['wideRoadCameraState'] = FrameReader(r.ecamera_paths()[sidx])
-    if len(r.dcamera_paths()) > sidx and r.dcamera_paths()[sidx] is not None:
+    if needs_driver_cam and len(r.dcamera_paths()) > sidx and r.dcamera_paths()[sidx] is not None:
       frs['driverCameraState'] = FrameReader(r.dcamera_paths()[sidx])
   else:
     lr = LogReader(f"cd:/{route.replace('|', '/')}/{sidx}/rlog.bz2")
-    frs = {
+    frs = {}
-      'roadCameraState': FrameReader(f"cd:/{route.replace('|', '/')}/{sidx}/fcamera.hevc"),
+    if needs_road_cam:
-      'driverCameraState': FrameReader(f"cd:/{route.replace('|', '/')}/{sidx}/dcamera.hevc"),
+      frs['roadCameraState'] = FrameReader(f"cd:/{route.replace('|', '/')}/{sidx}/fcamera.hevc")
-    }
       if next((True for m in lr if m.which() == "wideRoadCameraState"), False):
         frs['wideRoadCameraState'] = FrameReader(f"cd:/{route.replace('|', '/')}/{sidx}/ecamera.hevc")
+    if needs_driver_cam:
+      frs['driverCameraState'] = FrameReader(f"cd:/{route.replace('|', '/')}/{sidx}/dcamera.hevc")
 
   return lr, frs
 
 
 def regen_and_save(
-  route: str, sidx: int, daemons: Union[str, Iterable[str]] = "all", outdir: str = FAKEDATA,
+  route: str, sidx: int, processes: Union[str, Iterable[str]] = "all", outdir: str = FAKEDATA,
   upload: bool = False, use_route_meta: bool = False, disable_tqdm: bool = False
 ) -> str:
-  lr, frs = setup_data_readers(route, sidx, use_route_meta)
+  if not isinstance(processes, str) and not hasattr(processes, "__iter__"):
-  output_logs = regen_segment(lr, frs, daemons, disable_tqdm=disable_tqdm)
+    raise ValueError("whitelist_proc must be a string or iterable")
+
+  if processes != "all":
+    if isinstance(processes, str):
+      raise ValueError(f"Invalid value for processes: {processes}")
+
+    replayed_processes = []
+    for d in processes:
+      cfg = get_process_config(d)
+      replayed_processes.append(cfg)
+  else:
+    replayed_processes = CONFIGS
+
+  all_vision_pubs = {pub for cfg in replayed_processes for pub in cfg.vision_pubs}
+  lr, frs = setup_data_readers(route, sidx, use_route_meta,
+                               needs_driver_cam="driverCameraState" in all_vision_pubs,
+                               needs_road_cam="roadCameraState" in all_vision_pubs or "wideRoadCameraState" in all_vision_pubs)
+  output_logs = regen_segment(lr, frs, replayed_processes, disable_tqdm=disable_tqdm)
 
   log_dir = os.path.join(outdir, time.strftime("%Y-%m-%d--%H-%M-%S--0", time.gmtime()))
   rel_log_dir = os.path.relpath(log_dir)
@@ -110,5 +116,5 @@ if __name__ == "__main__":
   args = parser.parse_args()
 
   blacklist_set = set(args.blacklist_procs)
-  daemons = [p for p in args.whitelist_procs if p not in blacklist_set]
+  processes = [p for p in args.whitelist_procs if p not in blacklist_set]
-  regen_and_save(args.route, args.seg, daemons=daemons, upload=args.upload, outdir=args.outdir)
+  regen_and_save(args.route, args.seg, processes=processes, upload=args.upload, outdir=args.outdir)

selfdrive/test/process_replay/regen_all.py

@@ -29,15 +29,25 @@ def regen_job(segment, upload, disable_tqdm):
 
 
 if __name__ == "__main__":
+  all_cars = {car for car, _ in segments}
+
   parser = argparse.ArgumentParser(description="Generate new segments from old ones")
   parser.add_argument("-j", "--jobs", type=int, default=1)
   parser.add_argument("--no-upload", action="store_true")
+  parser.add_argument("--whitelist-cars", type=str, nargs="*", default=all_cars,
+                      help="Whitelist given cars from the test (e.g. HONDA)")
+  parser.add_argument("--blacklist-cars", type=str, nargs="*", default=[],
+                      help="Blacklist given cars from the test (e.g. HONDA)")
   args = parser.parse_args()
 
+  tested_cars = set(args.whitelist_cars) - set(args.blacklist_cars)
+  tested_cars = {c.upper() for c in tested_cars}
+  tested_segments = [(car, segment) for car, segment in segments if car in tested_cars]
+
   with concurrent.futures.ProcessPoolExecutor(max_workers=args.jobs) as pool:
-    p = pool.map(regen_job, segments, [not args.no_upload] * len(segments), [args.jobs > 1] * len(segments))
+    p = pool.map(regen_job, tested_segments, [not args.no_upload] * len(tested_segments), [args.jobs > 1] * len(tested_segments))
     msg = "Copy these new segments into test_processes.py:"
-    for seg in tqdm(p, desc="Generating segments", total=len(segments)):
+    for seg in tqdm(p, desc="Generating segments", total=len(tested_segments)):
       msg += "\n" + str(seg)
     print()
     print()

selfdrive/test/process_replay/test_processes.py

@@ -9,7 +9,7 @@ from typing import Any, DefaultDict, Dict
 
 from openpilot.selfdrive.car.car_helpers import interface_names
 from openpilot.selfdrive.test.openpilotci import get_url, upload_file
-from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs
+from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs, format_diff
 from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, PROC_REPLAY_DIR, FAKEDATA, check_openpilot_enabled, replay_process
 from openpilot.system.version import get_commit
 from openpilot.tools.lib.filereader import FileReader
@@ -115,45 +115,6 @@ def test_process(cfg, lr, segment, ref_log_path, new_log_path, ignore_fields=Non
     return str(e), log_msgs
 
 
-def format_diff(results, log_paths, ref_commit):
-  diff1, diff2 = "", ""
-  diff2 += f"***** tested against commit {ref_commit} *****\n"
-
-  failed = False
-  for segment, result in list(results.items()):
-    diff1 += f"***** results for segment {segment} *****\n"
-    diff2 += f"***** differences for segment {segment} *****\n"
-
-    for proc, diff in list(result.items()):
-      # long diff
-      diff2 += f"*** process: {proc} ***\n"
-      diff2 += f"\tref: {log_paths[segment][proc]['ref']}\n"
-      diff2 += f"\tnew: {log_paths[segment][proc]['new']}\n\n"
-
-      # short diff
-      diff1 += f"    {proc}\n"
-      if isinstance(diff, str):
-        diff1 += f"        ref: {log_paths[segment][proc]['ref']}\n"
-        diff1 += f"        new: {log_paths[segment][proc]['new']}\n\n"
-        diff1 += f"        {diff}\n"
-        failed = True
-      elif len(diff):
-        diff1 += f"        ref: {log_paths[segment][proc]['ref']}\n"
-        diff1 += f"        new: {log_paths[segment][proc]['new']}\n\n"
-
-        cnt: Dict[str, int] = {}
-        for d in diff:
-          diff2 += f"\t{str(d)}\n"
-
-          k = str(d[1])
-          cnt[k] = 1 if k not in cnt else cnt[k] + 1
-
-        for k, v in sorted(cnt.items()):
-          diff1 += f"        {k}: {v}\n"
-        failed = True
-  return diff1, diff2, failed
-
-
 if __name__ == "__main__":
   all_cars = {car for car, _ in segments}
   all_procs = {cfg.proc_name for cfg in CONFIGS if cfg.proc_name not in EXCLUDED_PROCS}

selfdrive/test/process_replay/test_regen.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+import unittest
+
+from parameterized import parameterized
+
+from openpilot.selfdrive.test.process_replay.regen import regen_segment
+from openpilot.selfdrive.test.process_replay.process_replay import check_openpilot_enabled, CONFIGS
+from openpilot.selfdrive.test.openpilotci import get_url
+from openpilot.tools.lib.logreader import LogReader
+from openpilot.tools.lib.framereader import FrameReader
+
+EXCLUDED_PROCESSES = {"dmonitoringd", "dmonitoringmodeld"}
+TESTED_SEGMENTS = [
+  ("PRIUS_C2", "0982d79ebb0de295|2021-01-04--17-13-21--13"), # TOYOTA PRIUS 2017:     NEO, pandaStateDEPRECATED, no peripheralState, sensorEventsDEPRECATED
+  # Enable these once regen on CI becomes faster or use them for different tests running controlsd in isolation
+  # ("MAZDA_C3", "bd6a637565e91581|2021-10-30--15-14-53--4"),  # MAZDA.CX9_2021:        TICI, incomplete managerState
+  # ("FORD_C3", "54827bf84c38b14f|2023-01-26--21-59-07--4"),   # FORD.BRONCO_SPORT_MK1: TICI
+]
+
+
+def ci_setup_data_readers(route, sidx):
+  lr = LogReader(get_url(route, sidx, "rlog"))
+  # dm disabled
+  frs = {
+    'roadCameraState': FrameReader(get_url(route, sidx, "fcamera")),
+  }
+  if next((True for m in lr if m.which() == "wideRoadCameraState"), False):
+    frs["wideRoadCameraState"] = FrameReader(get_url(route, sidx, "ecamera"))
+
+  return lr, frs
+
+
+class TestRegen(unittest.TestCase):
+  @parameterized.expand(TESTED_SEGMENTS)
+  def test_engaged(self, case_name, segment):
+    tested_procs = [p for p in CONFIGS if p.proc_name not in EXCLUDED_PROCESSES]
+
+    route, sidx = segment.rsplit("--", 1)
+    lr, frs = ci_setup_data_readers(route, sidx)
+    output_logs = regen_segment(lr, frs, processes=tested_procs, disable_tqdm=True)
+
+    engaged = check_openpilot_enabled(output_logs)
+    self.assertTrue(engaged, f"openpilot not engaged in {case_name}")
+
+
+if __name__=='__main__':
+  unittest.main()

selfdrive/test/process_replay/vision_meta.py

@@ -4,9 +4,9 @@ from openpilot.common.realtime import DT_MDL, DT_DMON
 from openpilot.common.transformations.camera import tici_f_frame_size, tici_d_frame_size, tici_e_frame_size, eon_f_frame_size, eon_d_frame_size
 
 VideoStreamMeta = namedtuple("VideoStreamMeta", ["camera_state", "encode_index", "stream", "dt", "frame_sizes"])
-ROAD_CAMERA_FRAME_SIZES = {"tici": tici_f_frame_size, "tizi": tici_f_frame_size, "eon": eon_f_frame_size}
+ROAD_CAMERA_FRAME_SIZES = {"tici": tici_f_frame_size, "tizi": tici_f_frame_size, "neo": eon_f_frame_size}
 WIDE_ROAD_CAMERA_FRAME_SIZES = {"tici": tici_e_frame_size, "tizi": tici_e_frame_size}
-DRIVER_FRAME_SIZES = {"tici": tici_d_frame_size, "tizi": tici_d_frame_size, "eon": eon_d_frame_size}
+DRIVER_FRAME_SIZES = {"tici": tici_d_frame_size, "tizi": tici_d_frame_size, "neo": eon_d_frame_size}
 VIPC_STREAM_METADATA = [
   # metadata: (state_msg_type, encode_msg_type, stream_type, dt, frame_sizes)
   ("roadCameraState", "roadEncodeIdx", VisionStreamType.VISION_STREAM_ROAD, DT_MDL, ROAD_CAMERA_FRAME_SIZES),

selfdrive/tombstoned.py

@@ -46,13 +46,10 @@ def get_apport_stacktrace(fn):
 
 
 def get_tombstones():
-  """Returns list of (filename, ctime) for all tombstones in /data/tombstones
+  """Returns list of (filename, ctime) for all crashlogs"""
-  and apport crashlogs in /var/crash"""
   files = []
-  for folder in [TOMBSTONE_DIR, APPORT_DIR]:
+  if os.path.exists(APPORT_DIR):
-    if os.path.exists(folder):
+    with os.scandir(APPORT_DIR) as d:
-      with os.scandir(folder) as d:
-
       # Loop over first 1000 directory entries
       for _, f in zip(range(1000), d, strict=False):
         if f.name.startswith("tombstone"):
@@ -143,7 +140,7 @@ def report_tombstone_apport(fn):
 
 
 def main() -> NoReturn:
-  sentry.init(sentry.SentryProject.SELFDRIVE_NATIVE)
+  should_report = sentry.init(sentry.SentryProject.SELFDRIVE_NATIVE)
 
   # Clear apport folder on start, otherwise duplicate crashes won't register
   clear_apport_folder()
@@ -153,6 +150,14 @@ def main() -> NoReturn:
     now_tombstones = set(get_tombstones())
 
     for fn, _ in (now_tombstones - initial_tombstones):
+      # clear logs if we're not interested in them
+      if not should_report:
+        try:
+          os.remove(fn)
+        except Exception:
+          pass
+        continue
+
       try:
         cloudlog.info(f"reporting new tombstone {fn}")
         if fn.endswith(".crash"):

selfdrive/ui/SConscript

@@ -20,7 +20,7 @@ if arch == "Darwin":
   qt_env['FRAMEWORKS'] += ['OpenCL']
 
 qt_util = qt_env.Library("qt_util", ["#selfdrive/ui/qt/api.cc", "#selfdrive/ui/qt/util.cc"], LIBS=base_libs)
-widgets_src = ["ui.cc", "qt/widgets/input.cc", "qt/widgets/drive_stats.cc", "qt/widgets/wifi.cc",
+widgets_src = ["ui.cc", "qt/widgets/input.cc", "qt/widgets/wifi.cc",
                "qt/widgets/ssh_keys.cc", "qt/widgets/toggle.cc", "qt/widgets/controls.cc",
                "qt/widgets/offroad_alerts.cc", "qt/widgets/prime.cc", "qt/widgets/keyboard.cc",
                "qt/widgets/scrollview.cc", "qt/widgets/cameraview.cc", "#third_party/qrcode/QrCode.cc",

selfdrive/ui/qt/home.cc

@@ -11,8 +11,6 @@
 
 #ifdef ENABLE_MAPS
 #include "selfdrive/ui/qt/maps/map_settings.h"
-#else
-#include "selfdrive/ui/qt/widgets/drive_stats.h"
 #endif
 
 // HomeWindow: the container for the offroad and onroad UIs
@@ -152,7 +150,7 @@ OffroadHome::OffroadHome(QWidget* parent) : QFrame(parent) {
 #ifdef ENABLE_MAPS
     left_widget->addWidget(new MapSettings);
 #else
-    left_widget->addWidget(new DriveStats);
+    left_widget->addWidget(new QWidget);
 #endif
     left_widget->addWidget(new PrimeAdWidget);
     left_widget->setStyleSheet("border-radius: 10px;");

selfdrive/ui/qt/onroad.cc

@@ -4,6 +4,7 @@
 #include <cmath>
 #include <map>
 #include <memory>
+#include <sstream>
 
 #include <QDebug>
 #include <QMouseEvent>

selfdrive/ui/qt/widgets/drive_stats.cc

@@ -1,97 +0,0 @@
-#include "selfdrive/ui/qt/widgets/drive_stats.h"
-
-#include <QDebug>
-#include <QGridLayout>
-#include <QJsonObject>
-#include <QVBoxLayout>
-
-#include "common/params.h"
-#include "selfdrive/ui/qt/request_repeater.h"
-#include "selfdrive/ui/qt/util.h"
-
-static QLabel* newLabel(const QString& text, const QString &type) {
-  QLabel* label = new QLabel(text);
-  label->setProperty("type", type);
-  return label;
-}
-
-DriveStats::DriveStats(QWidget* parent) : QFrame(parent) {
-  metric_ = Params().getBool("IsMetric");
-
-  QVBoxLayout* main_layout = new QVBoxLayout(this);
-  main_layout->setContentsMargins(50, 50, 50, 60);
-
-  auto add_stats_layouts = [=](const QString &title, StatsLabels& labels) {
-    QGridLayout* grid_layout = new QGridLayout;
-    grid_layout->setVerticalSpacing(10);
-    grid_layout->setContentsMargins(0, 10, 0, 10);
-
-    int row = 0;
-    grid_layout->addWidget(newLabel(title, "title"), row++, 0, 1, 3);
-    grid_layout->addItem(new QSpacerItem(0, 50), row++, 0, 1, 1);
-
-    grid_layout->addWidget(labels.routes = newLabel("0", "number"), row, 0, Qt::AlignLeft);
-    grid_layout->addWidget(labels.distance = newLabel("0", "number"), row, 1, Qt::AlignLeft);
-    grid_layout->addWidget(labels.hours = newLabel("0", "number"), row, 2, Qt::AlignLeft);
-
-    grid_layout->addWidget(newLabel((tr("Drives")), "unit"), row + 1, 0, Qt::AlignLeft);
-    grid_layout->addWidget(labels.distance_unit = newLabel(getDistanceUnit(), "unit"), row + 1, 1, Qt::AlignLeft);
-    grid_layout->addWidget(newLabel(tr("Hours"), "unit"), row + 1, 2, Qt::AlignLeft);
-
-    main_layout->addLayout(grid_layout);
-  };
-
-  add_stats_layouts(tr("ALL TIME"), all_);
-  main_layout->addStretch();
-  add_stats_layouts(tr("PAST WEEK"), week_);
-
-  if (auto dongleId = getDongleId()) {
-    QString url = CommaApi::BASE_URL + "/v1.1/devices/" + *dongleId + "/stats";
-    RequestRepeater* repeater = new RequestRepeater(this, url, "ApiCache_DriveStats", 30);
-    QObject::connect(repeater, &RequestRepeater::requestDone, this, &DriveStats::parseResponse);
-  }
-
-  setStyleSheet(R"(
-    DriveStats {
-      background-color: #333333;
-      border-radius: 10px;
-    }
-
-    QLabel[type="title"] { font-size: 51px; font-weight: 500; }
-    QLabel[type="number"] { font-size: 78px; font-weight: 500; }
-    QLabel[type="unit"] { font-size: 51px; font-weight: 300; color: #A0A0A0; }
-  )");
-}
-
-void DriveStats::updateStats() {
-  auto update = [=](const QJsonObject& obj, StatsLabels& labels) {
-    labels.routes->setText(QString::number((int)obj["routes"].toDouble()));
-    labels.distance->setText(QString::number(int(obj["distance"].toDouble() * (metric_ ? MILE_TO_KM : 1))));
-    labels.distance_unit->setText(getDistanceUnit());
-    labels.hours->setText(QString::number((int)(obj["minutes"].toDouble() / 60)));
-  };
-
-  QJsonObject json = stats_.object();
-  update(json["all"].toObject(), all_);
-  update(json["week"].toObject(), week_);
-}
-
-void DriveStats::parseResponse(const QString& response, bool success) {
-  if (!success) return;
-
-  QJsonDocument doc = QJsonDocument::fromJson(response.trimmed().toUtf8());
-  if (doc.isNull()) {
-    qDebug() << "JSON Parse failed on getting past drives statistics";
-    return;
-  }
-  stats_ = doc;
-  updateStats();
-}
-
-void DriveStats::showEvent(QShowEvent* event) {
-  bool metric = Params().getBool("IsMetric");
-  if (metric_ != metric) {
-    metric_ = metric;
-    updateStats();
-  }
-}

selfdrive/ui/qt/widgets/drive_stats.h

@@ -1,25 +0,0 @@
-#pragma once
-
-#include <QJsonDocument>
-#include <QLabel>
-
-class DriveStats : public QFrame {
-  Q_OBJECT
-
-public:
-  explicit DriveStats(QWidget* parent = 0);
-
-private:
-  void showEvent(QShowEvent *event) override;
-  void updateStats();
-  inline QString getDistanceUnit() const { return metric_ ? tr("KM") : tr("Miles"); }
-
-  bool metric_;
-  QJsonDocument stats_;
-  struct StatsLabels {
-    QLabel *routes, *distance, *distance_unit, *hours;
-  } all_, week_;
-
-private slots:
-  void parseResponse(const QString &response, bool success);
-};

selfdrive/ui/qt/widgets/input.h

@@ -26,7 +26,7 @@ class InputDialog : public DialogBase {
 
 public:
   explicit InputDialog(const QString &title, QWidget *parent, const QString &subtitle = "", bool secret = false);
-  static QString getText(const QString &title, QWidget *parent, const QString &substitle = "",
+  static QString getText(const QString &title, QWidget *parent, const QString &subtitle = "",
                          bool secret = false, int minLength = -1, const QString &defaultText = "");
   QString text();
   void setMessage(const QString &message, bool clearInputField = true);

selfdrive/ui/translations/main_ar.ts

@@ -274,33 +274,6 @@
         <translation>مراجعة</translation>
     </message>
 </context>
-<context>
-    <name>DriveStats</name>
-    <message>
-        <source>Drives</source>
-        <translation>القيادة</translation>
-    </message>
-    <message>
-        <source>Hours</source>
-        <translation>ساعات</translation>
-    </message>
-    <message>
-        <source>ALL TIME</source>
-        <translation>كامل الوقت</translation>
-    </message>
-    <message>
-        <source>PAST WEEK</source>
-        <translation>الأسبوع الماضي</translation>
-    </message>
-    <message>
-        <source>KM</source>
-        <translation>كم</translation>
-    </message>
-    <message>
-        <source>Miles</source>
-        <translation>ميل</translation>
-    </message>
-</context>
 <context>
     <name>DriverViewScene</name>
     <message>

selfdrive/ui/translations/main_de.ts

@@ -274,33 +274,6 @@
         <translation>Überprüfen</translation>
     </message>
 </context>
-<context>
-    <name>DriveStats</name>
-    <message>
-        <source>Drives</source>
-        <translation>Fahrten</translation>
-    </message>
-    <message>
-        <source>Hours</source>
-        <translation>Stunden</translation>
-    </message>
-    <message>
-        <source>ALL TIME</source>
-        <translation>Gesamtzeit</translation>
-    </message>
-    <message>
-        <source>PAST WEEK</source>
-        <translation>Letzte Woche</translation>
-    </message>
-    <message>
-        <source>KM</source>
-        <translation>KM</translation>
-    </message>
-    <message>
-        <source>Miles</source>
-        <translation>Meilen</translation>
-    </message>
-</context>
 <context>
     <name>DriverViewScene</name>
     <message>

selfdrive/ui/translations/main_fr.ts

@@ -274,33 +274,6 @@
         <translation>Désengager pour éteindre</translation>
     </message>
 </context>
-<context>
-    <name>DriveStats</name>
-    <message>
-        <source>Drives</source>
-        <translation>Trajets</translation>
-    </message>
-    <message>
-        <source>Hours</source>
-        <translation>Heures</translation>
-    </message>
-    <message>
-        <source>ALL TIME</source>
-        <translation>DEPUIS TOUJOURS</translation>
-    </message>
-    <message>
-        <source>PAST WEEK</source>
-        <translation>CETTE SEMAINE</translation>
-    </message>
-    <message>
-        <source>KM</source>
-        <translation>KM</translation>
-    </message>
-    <message>
-        <source>Miles</source>
-        <translation>Miles</translation>
-    </message>
-</context>
 <context>
     <name>DriverViewScene</name>
     <message>

selfdrive/ui/translations/main_ja.ts

@@ -274,33 +274,6 @@
         <translation>確認</translation>
     </message>
 </context>
-<context>
-    <name>DriveStats</name>
-    <message>
-        <source>Drives</source>
-        <translation>運転履歴</translation>
-    </message>
-    <message>
-        <source>Hours</source>
-        <translation>時間</translation>
-    </message>
-    <message>
-        <source>ALL TIME</source>
-        <translation>累計</translation>
-    </message>
-    <message>
-        <source>PAST WEEK</source>
-        <translation>先週</translation>
-    </message>
-    <message>
-        <source>KM</source>
-        <translation>km</translation>
-    </message>
-    <message>
-        <source>Miles</source>
-        <translation>マイル</translation>
-    </message>
-</context>
 <context>
     <name>DriverViewScene</name>
     <message>

selfdrive/ui/translations/main_ko.ts

@@ -274,33 +274,6 @@
         <translation>다시보기</translation>
     </message>
 </context>
-<context>
-    <name>DriveStats</name>
-    <message>
-        <source>Drives</source>
-        <translation>주행</translation>
-    </message>
-    <message>
-        <source>Hours</source>
-        <translation>시간</translation>
-    </message>
-    <message>
-        <source>ALL TIME</source>
-        <translation>전체</translation>
-    </message>
-    <message>
-        <source>PAST WEEK</source>
-        <translation>지난 주</translation>
-    </message>
-    <message>
-        <source>KM</source>
-        <translation>km</translation>
-    </message>
-    <message>
-        <source>Miles</source>
-        <translation>마일</translation>
-    </message>
-</context>
 <context>
     <name>DriverViewScene</name>
     <message>

selfdrive/ui/translations/main_pt-BR.ts

@@ -274,33 +274,6 @@
         <translation>Revisar</translation>
     </message>
 </context>
-<context>
-    <name>DriveStats</name>
-    <message>
-        <source>Drives</source>
-        <translation>Dirigidas</translation>
-    </message>
-    <message>
-        <source>Hours</source>
-        <translation>Horas</translation>
-    </message>
-    <message>
-        <source>ALL TIME</source>
-        <translation>TOTAL</translation>
-    </message>
-    <message>
-        <source>PAST WEEK</source>
-        <translation>SEMANA PASSADA</translation>
-    </message>
-    <message>
-        <source>KM</source>
-        <translation>KM</translation>
-    </message>
-    <message>
-        <source>Miles</source>
-        <translation>Milhas</translation>
-    </message>
-</context>
 <context>
     <name>DriverViewScene</name>
     <message>

selfdrive/ui/translations/main_th.ts

@@ -274,33 +274,6 @@
         <translation>ทบทวน</translation>
     </message>
 </context>
-<context>
-    <name>DriveStats</name>
-    <message>
-        <source>Drives</source>
-        <translation>การขับขี่</translation>
-    </message>
-    <message>
-        <source>Hours</source>
-        <translation>ชั่วโมง</translation>
-    </message>
-    <message>
-        <source>ALL TIME</source>
-        <translation>ทั้งหมด</translation>
-    </message>
-    <message>
-        <source>PAST WEEK</source>
-        <translation>สัปดาห์ที่ผ่านมา</translation>
-    </message>
-    <message>
-        <source>KM</source>
-        <translation>กิโลเมตร</translation>
-    </message>
-    <message>
-        <source>Miles</source>
-        <translation>ไมล์</translation>
-    </message>
-</context>
 <context>
     <name>DriverViewScene</name>
     <message>

selfdrive/ui/translations/main_tr.ts

@@ -274,33 +274,6 @@
         <translation type="unfinished"></translation>
     </message>
 </context>
-<context>
-    <name>DriveStats</name>
-    <message>
-        <source>Drives</source>
-        <translation>Sürücüler</translation>
-    </message>
-    <message>
-        <source>Hours</source>
-        <translation>Saat</translation>
-    </message>
-    <message>
-        <source>ALL TIME</source>
-        <translation>TÜM ZAMANLAR</translation>
-    </message>
-    <message>
-        <source>PAST WEEK</source>
-        <translation>GEÇEN HAFTA</translation>
-    </message>
-    <message>
-        <source>KM</source>
-        <translation>KM</translation>
-    </message>
-    <message>
-        <source>Miles</source>
-        <translation>Mil</translation>
-    </message>
-</context>
 <context>
     <name>DriverViewScene</name>
     <message>

selfdrive/ui/translations/main_zh-CHS.ts

@@ -274,33 +274,6 @@
         <translation>预览</translation>
     </message>
 </context>
-<context>
-    <name>DriveStats</name>
-    <message>
-        <source>Drives</source>
-        <translation>旅程数</translation>
-    </message>
-    <message>
-        <source>Hours</source>
-        <translation>小时</translation>
-    </message>
-    <message>
-        <source>ALL TIME</source>
-        <translation>全部</translation>
-    </message>
-    <message>
-        <source>PAST WEEK</source>
-        <translation>过去一周</translation>
-    </message>
-    <message>
-        <source>KM</source>
-        <translation>公里</translation>
-    </message>
-    <message>
-        <source>Miles</source>
-        <translation>英里</translation>
-    </message>
-</context>
 <context>
     <name>DriverViewScene</name>
     <message>

selfdrive/ui/translations/main_zh-CHT.ts

@@ -274,33 +274,6 @@
         <translation>回顧</translation>
     </message>
 </context>
-<context>
-    <name>DriveStats</name>
-    <message>
-        <source>Drives</source>
-        <translation>旅程</translation>
-    </message>
-    <message>
-        <source>Hours</source>
-        <translation>小時</translation>
-    </message>
-    <message>
-        <source>ALL TIME</source>
-        <translation>總共</translation>
-    </message>
-    <message>
-        <source>PAST WEEK</source>
-        <translation>上週</translation>
-    </message>
-    <message>
-        <source>KM</source>
-        <translation>公里</translation>
-    </message>
-    <message>
-        <source>Miles</source>
-        <translation>英里</translation>
-    </message>
-</context>
 <context>
     <name>DriverViewScene</name>
     <message>

system/hardware/hw.py

@@ -1,7 +1,7 @@
 import os
 from pathlib import Path
 
-from openpilot.selfdrive.hardware import PC
+from openpilot.system.hardware import PC
 
 class Paths:
   @staticmethod

system/loggerd/config.py

@@ -1,7 +1,7 @@
 import os
 from pathlib import Path
 from openpilot.system.hardware import PC
-from openpilot.selfdrive.hardware.hw import Paths
+from openpilot.system.hardware.hw import Paths
 
 
 CAMERA_FPS = 20

system/loggerd/tests/test_encoder.py

@@ -16,7 +16,7 @@ from openpilot.common.timeout import Timeout
 from openpilot.system.hardware import TICI
 from openpilot.selfdrive.manager.process_config import managed_processes
 from openpilot.tools.lib.logreader import LogReader
-from openpilot.selfdrive.hardware.hw import Paths
+from openpilot.system.hardware.hw import Paths
 
 SEGMENT_LENGTH = 2
 FULL_SIZE = 2507572

system/sensord/pigeond.py

@@ -164,12 +164,12 @@ def initialize_pigeon(pigeon: TTYPigeon) -> bool:
       pigeon.send_with_ack(b"\xB5\x62\x06\x08\x06\x00\x64\x00\x01\x00\x00\x00\x79\x10")
 
       # UBX-CFG-NAV5 (0x06 0x24)
-      pigeon.send_with_ack(b"\xB5\x62\x06\x24\x24\x00\x05\x00\x04\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x5A\x63") # noqa: E501
+      pigeon.send_with_ack(b"\xB5\x62\x06\x24\x24\x00\x05\x00\x04\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x5A\x63")
 
       # UBX-CFG-ODO (0x06 0x1E)
       pigeon.send_with_ack(b"\xB5\x62\x06\x1E\x14\x00\x00\x00\x00\x00\x01\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3C\x37")
       pigeon.send_with_ack(b"\xB5\x62\x06\x39\x08\x00\xFF\xAD\x62\xAD\x1E\x63\x00\x00\x83\x0C")
-      pigeon.send_with_ack(b"\xB5\x62\x06\x23\x28\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x56\x24") # noqa: E501
+      pigeon.send_with_ack(b"\xB5\x62\x06\x23\x28\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x56\x24")
 
       # UBX-CFG-NAV5 (0x06 0x24)
       pigeon.send_with_ack(b"\xB5\x62\x06\x24\x00\x00\x2A\x84")

tools/cabana/chart/chart.cc

@@ -169,7 +169,7 @@ void ChartView::msgUpdated(MessageId id) {
 }
 
 void ChartView::manageSignals() {
-  SignalSelector dlg(tr("Mange Chart"), this);
+  SignalSelector dlg(tr("Manage Chart"), this);
   for (auto &s : sigs) {
     dlg.addSelected(s.msg_id, s.sig);
   }

tools/cabana/dbc/dbcfile.cc

@@ -107,7 +107,8 @@ void DBCFile::parse(const QString &content) {
   int multiplexor_cnt = 0;
   while (!stream.atEnd()) {
     ++line_num;
-    line = stream.readLine().trimmed();
+    QString raw_line = stream.readLine();
+    line = raw_line.trimmed();
     if (line.startsWith("BO_ ")) {
       multiplexor_cnt = 0;
       auto match = bo_regexp.match(line);
@@ -170,7 +171,7 @@ void DBCFile::parse(const QString &content) {
       }
     } else if (line.startsWith("CM_ BO_")) {
       if (!line.endsWith("\";")) {
-        int pos = stream.pos() - line.length() - 1;
+        int pos = stream.pos() - raw_line.length() - 1;
         line = content.mid(pos, content.indexOf("\";", pos));
       }
       auto match = msg_comment_regexp.match(line);
@@ -180,7 +181,7 @@ void DBCFile::parse(const QString &content) {
       }
     } else if (line.startsWith("CM_ SG_ ")) {
       if (!line.endsWith("\";")) {
-        int pos = stream.pos() - line.length() - 1;
+        int pos = stream.pos() - raw_line.length() - 1;
         line = content.mid(pos, content.indexOf("\";", pos));
       }
       auto match = sg_comment_regexp.match(line);

tools/cabana/messageswidget.cc

@@ -169,6 +169,8 @@ QVariant MessageListModel::headerData(int section, Qt::Orientation orientation,
 }
 
 QVariant MessageListModel::data(const QModelIndex &index, int role) const {
+  if (!index.isValid() || index.row() >= msgs.size()) return {};
+
   const auto &id = msgs[index.row()];
   auto &can_data = can->lastMessage(id);
 

tools/cabana/tests/test_cabana.cc

@@ -69,7 +69,7 @@ TEST_CASE("Parse can messages") {
   }
 }
 
-TEST_CASE("Parse dbc") {
+TEST_CASE("parse_dbc") {
   QString content = R"(
 BO_ 160 message_1: 8 EON
   SG_ signal_1 : 0|12@1+ (1,0) [0|4095] "unit"  XXX
@@ -113,7 +113,7 @@ CM_ SG_ 160 signal_2 "multiple line comment
   REQUIRE(sig_1->val_desc[2] == std::pair<double, QString>{2, "fault"});
 
   auto &sig_2 = msg->sigs[1];
-  REQUIRE(sig_2->comment == "multiple line comment\n1\n2");
+  REQUIRE(sig_2->comment == "multiple line comment \n1\n2");
 
   // multiplexed signals
   msg = file.msg(162);

tools/lib/filereader.py

@@ -3,9 +3,13 @@ from openpilot.tools.lib.url_file import URLFile
 
 DATA_ENDPOINT = os.getenv("DATA_ENDPOINT", "http://data-raw.comma.internal/")
 
-def FileReader(fn, debug=False):
+def resolve_name(fn):
   if fn.startswith("cd:/"):
-    fn = fn.replace("cd:/", DATA_ENDPOINT)
+    return fn.replace("cd:/", DATA_ENDPOINT)
+  return fn
+
+def FileReader(fn, debug=False):
+  fn = resolve_name(fn)
   if fn.startswith(("http://", "https://")):
     return URLFile(fn, debug=debug)
   return open(fn, "rb")

tools/lib/framereader.py

@@ -14,9 +14,10 @@ from lru import LRU
 import _io
 from openpilot.tools.lib.cache import cache_path_for_file_path, DEFAULT_CACHE_DIR
 from openpilot.tools.lib.exceptions import DataUnreadableError
+from openpilot.tools.lib.vidindex import hevc_index
 from openpilot.common.file_helpers import atomic_write_in_dir
 
-from openpilot.tools.lib.filereader import FileReader
+from openpilot.tools.lib.filereader import FileReader, resolve_name
 
 HEVC_SLICE_B = 0
 HEVC_SLICE_P = 1
@@ -59,6 +60,7 @@ def fingerprint_video(fn):
 
 
 def ffprobe(fn, fmt=None):
+  fn = resolve_name(fn)
   cmd = ["ffprobe",
          "-v", "quiet",
          "-print_format", "json",
@@ -75,31 +77,6 @@ def ffprobe(fn, fmt=None):
   return json.loads(ffprobe_output)
 
 
-def vidindex(fn, typ):
-  vidindex_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "vidindex")
-  vidindex = os.path.join(vidindex_dir, "vidindex")
-
-  subprocess.check_call(["make"], cwd=vidindex_dir, stdout=subprocess.DEVNULL)
-
-  with tempfile.NamedTemporaryFile() as prefix_f, \
-       tempfile.NamedTemporaryFile() as index_f:
-    try:
-      subprocess.check_call([vidindex, typ, fn, prefix_f.name, index_f.name])
-    except subprocess.CalledProcessError as e:
-      raise DataUnreadableError(f"vidindex failed on file {fn}") from e
-    with open(index_f.name, "rb") as f:
-      index = f.read()
-    with open(prefix_f.name, "rb") as f:
-      prefix = f.read()
-
-  index = np.frombuffer(index, np.uint32).reshape(-1, 2)
-
-  assert index[-1, 0] == 0xFFFFFFFF
-  assert index[-1, 1] == os.path.getsize(fn)
-
-  return index, prefix
-
-
 def cache_fn(func):
   @wraps(func)
   def cache_inner(fn, *args, **kwargs):
@@ -114,7 +91,6 @@ def cache_fn(func):
         cache_value = pickle.load(cache_file)
     else:
       cache_value = func(fn, *args, **kwargs)
-
       if cache_path:
         with atomic_write_in_dir(cache_path, mode="wb", overwrite=True) as cache_file:
           pickle.dump(cache_value, cache_file, -1)
@@ -125,13 +101,13 @@ def cache_fn(func):
 
 
 @cache_fn
-def index_stream(fn, typ):
+def index_stream(fn, ft):
-  assert typ in ("hevc", )
+  if ft != FrameType.h265_stream:
+    raise NotImplementedError("Only h265 supported")
 
-  with FileReader(fn) as f:
+  frame_types, dat_len, prefix = hevc_index(fn)
-    assert os.path.exists(f.name), fn
+  index = np.array(frame_types + [(0xFFFFFFFF, dat_len)], dtype=np.uint32)
-    index, prefix = vidindex(f.name, typ)
+  probe = ffprobe(fn, "hevc")
-    probe = ffprobe(f.name, typ)
 
   return {
     'index': index,
@@ -140,42 +116,8 @@ def index_stream(fn, typ):
   }
 
 
-def index_videos(camera_paths, cache_dir=DEFAULT_CACHE_DIR):
-  """Requires that paths in camera_paths are contiguous and of the same type."""
-  if len(camera_paths) < 1:
-    raise ValueError("must provide at least one video to index")
-
-  frame_type = fingerprint_video(camera_paths[0])
-  for fn in camera_paths:
-    index_video(fn, frame_type, cache_dir)
-
-
-def index_video(fn, frame_type=None, cache_dir=DEFAULT_CACHE_DIR):
-  cache_path = cache_path_for_file_path(fn, cache_dir)
-
-  if os.path.exists(cache_path):
-    return
-
-  if frame_type is None:
-    frame_type = fingerprint_video(fn[0])
-
-  if frame_type == FrameType.h265_stream:
-    index_stream(fn, "hevc", cache_dir=cache_dir)
-  else:
-    raise NotImplementedError("Only h265 supported")
-
-
 def get_video_index(fn, frame_type, cache_dir=DEFAULT_CACHE_DIR):
-  cache_path = cache_path_for_file_path(fn, cache_dir)
+  return index_stream(fn, frame_type, cache_dir=cache_dir)
-
-  if not os.path.exists(cache_path):
-    index_video(fn, frame_type, cache_dir)
-
-  if not os.path.exists(cache_path):
-    return None
-  with open(cache_path, "rb") as cache_file:
-    return pickle.load(cache_file)
-
 
 def read_file_check_size(f, sz, cookie):
   buff = bytearray(sz)

tools/lib/tests/test_caching.py

@@ -1,6 +1,12 @@
 #!/usr/bin/env python3
 import os
 import unittest
+
+from pathlib import Path
+from parameterized import parameterized
+from unittest import mock
+
+from openpilot.system.hardware.hw import Paths
 from openpilot.tools.lib.url_file import URLFile
 
 
@@ -59,6 +65,34 @@ class TestFileDownload(unittest.TestCase):
     self.compare_loads(large_file_url, length - 100, 100)
     self.compare_loads(large_file_url)
 
+  @parameterized.expand([(True, ), (False, )])
+  def test_recover_from_missing_file(self, cache_enabled):
+    os.environ["FILEREADER_CACHE"] = "1" if cache_enabled else "0"
+
+    file_url = "http://localhost:5001/test.png"
+
+    file_exists = False
+
+    def get_length_online_mock(self):
+      if file_exists:
+        return 4
+      return -1
+
+    patch_length = mock.patch.object(URLFile, "get_length_online", get_length_online_mock)
+    patch_length.start()
+    try:
+      length = URLFile(file_url).get_length()
+      self.assertEqual(length, -1)
+
+      file_exists = True
+      length = URLFile(file_url).get_length()
+      self.assertEqual(length, 4)
+    finally:
+      tempfile_length = Path(Paths.download_cache_root()) / "ba2119904385654cb0105a2da174875f8e7648db175f202ecae6d6428b0e838f_length"
+      if tempfile_length.exists():
+        tempfile_length.unlink()
+      patch_length.stop()
+
 
 if __name__ == "__main__":
   unittest.main()

tools/lib/url_file.py

@@ -1,8 +1,6 @@
 import os
 import time
-import tempfile
 import threading
-import urllib.parse
 import pycurl
 from hashlib import sha256
 from io import BytesIO
@@ -37,6 +35,7 @@ class URLFile:
       self._curl = self._tlocal.curl
     except AttributeError:
       self._curl = self._tlocal.curl = pycurl.Curl()
+    if not self._force_download:
       mkdirs_exists_ok(Paths.download_cache_root())
 
   def __enter__(self):
@@ -65,15 +64,16 @@ class URLFile:
   def get_length(self):
     if self._length is not None:
       return self._length
+
     file_length_path = os.path.join(Paths.download_cache_root(), hash_256(self._url) + "_length")
-    if os.path.exists(file_length_path) and not self._force_download:
+    if not self._force_download and os.path.exists(file_length_path):
       with open(file_length_path) as file_length:
         content = file_length.read()
         self._length = int(content)
         return self._length
 
     self._length = self.get_length_online()
-    if not self._force_download:
+    if not self._force_download and self._length != -1:
       with atomic_write_in_dir(file_length_path, mode="w") as file_length:
         file_length.write(str(self._length))
     return self._length
@@ -173,24 +173,4 @@ class URLFile:
 
   @property
   def name(self):
-    """Returns a local path to file with the URLFile's contents.
+    return self._url
-
-       This can be used to interface with modules that require local files.
-    """
-    if self._local_file is None:
-      _, ext = os.path.splitext(urllib.parse.urlparse(self._url).path)
-      local_fd, local_path = tempfile.mkstemp(suffix=ext)
-      try:
-        os.write(local_fd, self.read())
-        local_file = open(local_path, "rb")
-      except Exception:
-        os.remove(local_path)
-        raise
-      finally:
-        os.close(local_fd)
-
-      self._local_file = local_file
-      self.read = self._local_file.read
-      self.seek = self._local_file.seek
-
-    return self._local_file.name

tools/lib/vidindex.py

@@ -0,0 +1,312 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import struct
+from enum import IntEnum
+from typing import Tuple
+
+from openpilot.tools.lib.filereader import FileReader
+
+DEBUG = int(os.getenv("DEBUG", "0"))
+
+# compare to ffmpeg parsing
+# ffmpeg -i <input.hevc> -c copy -bsf:v trace_headers -f null - 2>&1 | grep -B4 -A32 '] 0 '
+
+# H.265 specification
+# https://www.itu.int/rec/dologin_pub.asp?lang=e&id=T-REC-H.265-201802-S!!PDF-E&type=items
+
+NAL_UNIT_START_CODE = b"\x00\x00\x01"
+NAL_UNIT_START_CODE_SIZE = len(NAL_UNIT_START_CODE)
+NAL_UNIT_HEADER_SIZE = 2
+
+class HevcNalUnitType(IntEnum):
+  TRAIL_N = 0         # RBSP structure: slice_segment_layer_rbsp( )
+  TRAIL_R = 1         # RBSP structure: slice_segment_layer_rbsp( )
+  TSA_N = 2           # RBSP structure: slice_segment_layer_rbsp( )
+  TSA_R = 3           # RBSP structure: slice_segment_layer_rbsp( )
+  STSA_N = 4          # RBSP structure: slice_segment_layer_rbsp( )
+  STSA_R = 5          # RBSP structure: slice_segment_layer_rbsp( )
+  RADL_N = 6          # RBSP structure: slice_segment_layer_rbsp( )
+  RADL_R = 7          # RBSP structure: slice_segment_layer_rbsp( )
+  RASL_N = 8          # RBSP structure: slice_segment_layer_rbsp( )
+  RASL_R = 9          # RBSP structure: slice_segment_layer_rbsp( )
+  RSV_VCL_N10 = 10
+  RSV_VCL_R11 = 11
+  RSV_VCL_N12 = 12
+  RSV_VCL_R13 = 13
+  RSV_VCL_N14 = 14
+  RSV_VCL_R15 = 15
+  BLA_W_LP = 16       # RBSP structure: slice_segment_layer_rbsp( )
+  BLA_W_RADL = 17     # RBSP structure: slice_segment_layer_rbsp( )
+  BLA_N_LP = 18       # RBSP structure: slice_segment_layer_rbsp( )
+  IDR_W_RADL = 19     # RBSP structure: slice_segment_layer_rbsp( )
+  IDR_N_LP = 20       # RBSP structure: slice_segment_layer_rbsp( )
+  CRA_NUT = 21        # RBSP structure: slice_segment_layer_rbsp( )
+  RSV_IRAP_VCL22 = 22
+  RSV_IRAP_VCL23 = 23
+  RSV_VCL24 = 24
+  RSV_VCL25 = 25
+  RSV_VCL26 = 26
+  RSV_VCL27 = 27
+  RSV_VCL28 = 28
+  RSV_VCL29 = 29
+  RSV_VCL30 = 30
+  RSV_VCL31 = 31
+  VPS_NUT = 32        # RBSP structure: video_parameter_set_rbsp( )
+  SPS_NUT = 33        # RBSP structure: seq_parameter_set_rbsp( )
+  PPS_NUT = 34        # RBSP structure: pic_parameter_set_rbsp( )
+  AUD_NUT = 35
+  EOS_NUT = 36
+  EOB_NUT = 37
+  FD_NUT = 38
+  PREFIX_SEI_NUT = 39
+  SUFFIX_SEI_NUT = 40
+  RSV_NVCL41 = 41
+  RSV_NVCL42 = 42
+  RSV_NVCL43 = 43
+  RSV_NVCL44 = 44
+  RSV_NVCL45 = 45
+  RSV_NVCL46 = 46
+  RSV_NVCL47 = 47
+  UNSPEC48 = 48
+  UNSPEC49 = 49
+  UNSPEC50 = 50
+  UNSPEC51 = 51
+  UNSPEC52 = 52
+  UNSPEC53 = 53
+  UNSPEC54 = 54
+  UNSPEC55 = 55
+  UNSPEC56 = 56
+  UNSPEC57 = 57
+  UNSPEC58 = 58
+  UNSPEC59 = 59
+  UNSPEC60 = 60
+  UNSPEC61 = 61
+  UNSPEC62 = 62
+  UNSPEC63 = 63
+
+# B.2.2 Byte stream NAL unit semantics
+# - The nal_unit_type within the nal_unit( ) syntax structure is equal to VPS_NUT, SPS_NUT or PPS_NUT.
+# - The byte stream NAL unit syntax structure contains the first NAL unit of an access unit in decoding
+#   order, as specified in clause 7.4.2.4.4.
+HEVC_PARAMETER_SET_NAL_UNITS = (
+  HevcNalUnitType.VPS_NUT,
+  HevcNalUnitType.SPS_NUT,
+  HevcNalUnitType.PPS_NUT,
+)
+
+# 3.29 coded slice segment NAL unit: A NAL unit that has nal_unit_type in the range of TRAIL_N to RASL_R,
+# inclusive, or in the range of BLA_W_LP to RSV_IRAP_VCL23, inclusive, which indicates that the NAL unit
+# contains a coded slice segment
+HEVC_CODED_SLICE_SEGMENT_NAL_UNITS = (
+  HevcNalUnitType.TRAIL_N,
+  HevcNalUnitType.TRAIL_R,
+  HevcNalUnitType.TSA_N,
+  HevcNalUnitType.TSA_R,
+  HevcNalUnitType.STSA_N,
+  HevcNalUnitType.STSA_R,
+  HevcNalUnitType.RADL_N,
+  HevcNalUnitType.RADL_R,
+  HevcNalUnitType.RASL_N,
+  HevcNalUnitType.RASL_R,
+  HevcNalUnitType.BLA_W_LP,
+  HevcNalUnitType.BLA_W_RADL,
+  HevcNalUnitType.BLA_N_LP,
+  HevcNalUnitType.IDR_W_RADL,
+  HevcNalUnitType.IDR_N_LP,
+  HevcNalUnitType.CRA_NUT,
+)
+
+class VideoFileInvalid(Exception):
+  pass
+
+def get_ue(dat: bytes, start_idx: int, skip_bits: int) -> Tuple[int, int]:
+  prefix_val = 0
+  prefix_len = 0
+  suffix_val = 0
+  suffix_len = 0
+
+  i = start_idx
+  while i < len(dat):
+    j = 7
+    while j >= 0:
+      if skip_bits > 0:
+        skip_bits -= 1
+      elif prefix_val == 0:
+        prefix_val = (dat[i] >> j) & 1
+        prefix_len += 1
+      else:
+        suffix_val = (suffix_val << 1) | ((dat[i] >> j) & 1)
+        suffix_len += 1
+      j -= 1
+
+      if prefix_val == 1 and prefix_len - 1 == suffix_len:
+        val = 2**(prefix_len-1) - 1 + suffix_val
+        size = prefix_len + suffix_len
+        return val, size
+    i += 1
+
+  raise VideoFileInvalid("invalid exponential-golomb code")
+
+def require_nal_unit_start(dat: bytes, nal_unit_start: int) -> None:
+  if nal_unit_start < 1:
+    raise ValueError("start index must be greater than zero")
+
+  if dat[nal_unit_start:nal_unit_start + NAL_UNIT_START_CODE_SIZE] != NAL_UNIT_START_CODE:
+    raise VideoFileInvalid("data must begin with start code")
+
+def get_hevc_nal_unit_length(dat: bytes, nal_unit_start: int) -> int:
+  try:
+    pos = dat.index(NAL_UNIT_START_CODE, nal_unit_start + NAL_UNIT_START_CODE_SIZE)
+  except ValueError:
+    pos = -1
+
+  # length of NAL unit is byte count up to next NAL unit start index
+  nal_unit_len = (pos if pos != -1 else len(dat)) - nal_unit_start
+  if DEBUG:
+    print("  nal_unit_len:", nal_unit_len)
+  return nal_unit_len
+
+def get_hevc_nal_unit_type(dat: bytes, nal_unit_start: int) -> HevcNalUnitType:
+  # 7.3.1.2 NAL unit header syntax
+  # nal_unit_header( ) {    // descriptor
+  #   forbidden_zero_bit    f(1)
+  #   nal_unit_type         u(6)
+  #   nuh_layer_id          u(6)
+  #   nuh_temporal_id_plus1 u(3)
+  # }
+  header_start = nal_unit_start + NAL_UNIT_START_CODE_SIZE
+  nal_unit_header = dat[header_start:header_start + NAL_UNIT_HEADER_SIZE]
+  if len(nal_unit_header) != 2:
+    raise VideoFileInvalid("data to short to contain nal unit header")
+  nal_unit_type = HevcNalUnitType((nal_unit_header[0] >> 1) & 0x3F)
+  if DEBUG:
+    print("  nal_unit_type:", nal_unit_type.name, f"({nal_unit_type.value})")
+  return nal_unit_type
+
+def get_hevc_slice_type(dat: bytes, nal_unit_start: int, nal_unit_type: HevcNalUnitType) -> Tuple[int, bool]:
+  # 7.3.2.9 Slice segment layer RBSP syntax
+  # slice_segment_layer_rbsp( ) {
+  #   slice_segment_header( )
+  #   slice_segment_data( )
+  #   rbsp_slice_segment_trailing_bits( )
+  # }
+  # ...
+  # 7.3.6.1 General slice segment header syntax
+  # slice_segment_header( ) {                                             // descriptor
+  #   first_slice_segment_in_pic_flag                                     u(1)
+  #   if( nal_unit_type >= BLA_W_LP && nal_unit_type <= RSV_IRAP_VCL23 )
+  #     no_output_of_prior_pics_flag                                      u(1)
+  #   slice_pic_parameter_set_id                                         ue(v)
+  #   if( !first_slice_segment_in_pic_flag ) {
+  #     if( dependent_slice_segments_enabled_flag )
+  #       dependent_slice_segment_flag                                    u(1)
+  #     slice_segment_address                                             u(v)
+  #   }
+  #   if( !dependent_slice_segment_flag ) {
+  #     for( i = 0; i < num_extra_slice_header_bits; i++ )
+  #       slice_reserved_flag[ i ]                                        u(1)
+  #     slice_type                                                       ue(v)
+  # ...
+
+  rbsp_start = nal_unit_start + NAL_UNIT_START_CODE_SIZE + NAL_UNIT_HEADER_SIZE
+  skip_bits = 0
+
+  # 7.4.7.1 General slice segment header semantics
+  # first_slice_segment_in_pic_flag equal to 1 specifies that the slice segment is the first slice segment of the picture in
+  # decoding order. first_slice_segment_in_pic_flag equal to 0 specifies that the slice segment is not the first slice segment
+  # of the picture in decoding order.
+  is_first_slice = dat[rbsp_start] >> 7 & 1 == 1
+  if not is_first_slice:
+    # TODO: parse dependent_slice_segment_flag and slice_segment_address and get real slice_type
+    # for now since we don't use it return -1 for slice_type
+    return (-1, is_first_slice)
+  skip_bits += 1 # skip past first_slice_segment_in_pic_flag
+
+  if nal_unit_type >= HevcNalUnitType.BLA_W_LP and nal_unit_type <= HevcNalUnitType.RSV_IRAP_VCL23:
+    # 7.4.7.1 General slice segment header semantics
+    # no_output_of_prior_pics_flag affects the output of previously-decoded pictures in the decoded picture buffer after the
+    # decoding of an IDR or a BLA picture that is not the first picture in the bitstream as specified in Annex C.
+    skip_bits += 1 # skip past no_output_of_prior_pics_flag
+
+  # 7.4.7.1 General slice segment header semantics
+  # slice_pic_parameter_set_id specifies the value of pps_pic_parameter_set_id for the PPS in use.
+  # The value of slice_pic_parameter_set_id shall be in the range of 0 to 63, inclusive.
+  _, size = get_ue(dat, rbsp_start, skip_bits)
+  skip_bits += size # skip past slice_pic_parameter_set_id
+
+  # 7.4.3.3.1 General picture parameter set RBSP semanal_unit_lenntics
+  # num_extra_slice_header_bits specifies the number of extra slice header bits that are present in the slice header RBSP
+  # for coded pictures referring to the PPS. The value of num_extra_slice_header_bits shall be in the range of 0 to 2, inclusive,
+  # in bitstreams conforming to this version of this Specification. Other values for num_extra_slice_header_bits are reserved
+  # for future use by ITU-T | ISO/IEC. However, decoders shall allow num_extra_slice_header_bits to have any value.
+  # TODO: get from PPS_NUT pic_parameter_set_rbsp( ) for corresponding slice_pic_parameter_set_id
+  num_extra_slice_header_bits = 0
+  skip_bits += num_extra_slice_header_bits
+
+  # 7.4.7.1 General slice segment header semantics
+  # slice_type specifies the coding type of the slice according to Table 7-7.
+  # Table 7-7 - Name association to slice_type
+  # slice_type | Name of slice_type
+  #     0      | B (B slice)
+  #     1      | P (P slice)
+  #     2      | I (I slice)
+  # unsigned integer 0-th order Exp-Golomb-coded syntax element with the left bit first
+  slice_type, _ = get_ue(dat, rbsp_start, skip_bits)
+  if DEBUG:
+    print("  slice_type:", slice_type, f"(first slice: {is_first_slice})")
+  if slice_type > 2:
+    raise VideoFileInvalid("slice_type must be 0, 1, or 2")
+  return slice_type, is_first_slice
+
+def hevc_index(hevc_file_name: str, allow_corrupt: bool=False) -> Tuple[list, int, bytes]:
+  with FileReader(hevc_file_name) as f:
+    dat = f.read()
+
+  if len(dat) < NAL_UNIT_START_CODE_SIZE + 1:
+    raise VideoFileInvalid("data is too short")
+
+  if dat[0] != 0x00:
+    raise VideoFileInvalid("first byte must be 0x00")
+
+  prefix_dat = b""
+  frame_types = list()
+
+  i = 1 # skip past first byte 0x00
+  try:
+    while i < len(dat):
+      require_nal_unit_start(dat, i)
+      nal_unit_len = get_hevc_nal_unit_length(dat, i)
+      nal_unit_type = get_hevc_nal_unit_type(dat, i)
+      if nal_unit_type in HEVC_PARAMETER_SET_NAL_UNITS:
+        prefix_dat += dat[i:i+nal_unit_len]
+      elif nal_unit_type in HEVC_CODED_SLICE_SEGMENT_NAL_UNITS:
+        slice_type, is_first_slice = get_hevc_slice_type(dat, i, nal_unit_type)
+        if is_first_slice:
+          frame_types.append((slice_type, i))
+      i += nal_unit_len
+  except Exception as e:
+    if not allow_corrupt:
+      raise
+    print(f"ERROR: NAL unit skipped @ {i}\n", str(e))
+
+  return frame_types, len(dat), prefix_dat
+
+def main() -> None:
+  parser = argparse.ArgumentParser()
+  parser.add_argument("input_file", type=str)
+  parser.add_argument("output_prefix_file", type=str)
+  parser.add_argument("output_index_file", type=str)
+  args = parser.parse_args()
+
+  frame_types, dat_len, prefix_dat = hevc_index(args.input_file)
+  with open(args.output_prefix_file, "wb") as f:
+    f.write(prefix_dat)
+
+  with open(args.output_index_file, "wb") as f:
+    for ft, fp in frame_types:
+      f.write(struct.pack("<II", ft, fp))
+    f.write(struct.pack("<II", 0xFFFFFFFF, dat_len))
+
+if __name__ == "__main__":
+  main()

tools/lib/vidindex/.gitignore

@@ -1 +0,0 @@
-vidindex

tools/lib/vidindex/Makefile

@@ -1,6 +0,0 @@
-CC := gcc
-
-vidindex: bitstream.c bitstream.h vidindex.c
-	$(eval $@_TMP := $(shell mktemp))
-	$(CC) -std=c99 bitstream.c vidindex.c -o $($@_TMP)
-	mv $($@_TMP) $@

tools/lib/vidindex/bitstream.c

@@ -1,118 +0,0 @@
-#include "./bitstream.h"
-
-#include <stdbool.h>
-#include <assert.h>
-
-static const uint32_t BS_MASKS[33] = {
-    0,           0x1L,        0x3L,       0x7L,       0xFL,       0x1FL,
-    0x3FL,       0x7FL,       0xFFL,      0x1FFL,     0x3FFL,     0x7FFL,
-    0xFFFL,      0x1FFFL,     0x3FFFL,    0x7FFFL,    0xFFFFL,    0x1FFFFL,
-    0x3FFFFL,    0x7FFFFL,    0xFFFFFL,   0x1FFFFFL,  0x3FFFFFL,  0x7FFFFFL,
-    0xFFFFFFL,   0x1FFFFFFL,  0x3FFFFFFL, 0x7FFFFFFL, 0xFFFFFFFL, 0x1FFFFFFFL,
-    0x3FFFFFFFL, 0x7FFFFFFFL, 0xFFFFFFFFL};
-
-void bs_init(struct bitstream* bs, const uint8_t* buffer, size_t input_size) {
-  bs->buffer_ptr = buffer;
-  bs->buffer_end = buffer + input_size;
-  bs->value = 0;
-  bs->pos = 0;
-  bs->shift = 8;
-  bs->size = input_size * 8;
-}
-
-uint32_t bs_get(struct bitstream* bs, int n) {
-  if (n > 32)
-    return 0;
-
-  bs->pos += n;
-  bs->shift += n;
-  while (bs->shift > 8) {
-    if (bs->buffer_ptr < bs->buffer_end) {
-      bs->value <<= 8;
-      bs->value |= *bs->buffer_ptr++;
-      bs->shift -= 8;
-    } else {
-      bs_seek(bs, bs->pos - n);
-      return 0;
-      // bs->value <<= 8;
-      // bs->shift -= 8;
-    }
-  }
-  return (bs->value >> (8 - bs->shift)) & BS_MASKS[n];
-}
-
-void bs_seek(struct bitstream* bs, size_t new_pos) {
-  bs->pos = (new_pos / 32) * 32;
-  bs->shift = 8;
-  bs->value = 0;
-  bs_get(bs, new_pos % 32);
-}
-
-uint32_t bs_peek(struct bitstream* bs, int n) {
-  struct bitstream bak = *bs;
-  return bs_get(&bak, n);
-}
-
-size_t bs_remain(struct bitstream* bs) {
-  return bs->size - bs->pos;
-}
-
-int bs_eof(struct bitstream* bs) {
-  return bs_remain(bs) == 0;
-}
-
-uint32_t bs_ue(struct bitstream* bs) {
-  static const uint8_t exp_golomb_bits[256] = {
-      8, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3,
-      3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-      2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1,
-      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
-      1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-  };
-  uint32_t bits, read = 0;
-  int bits_left;
-  uint8_t coded;
-  int done = 0;
-  bits = 0;
-  // we want to read 8 bits at a time - if we don't have 8 bits,
-  // read what's left, and shift.  The exp_golomb_bits calc remains the
-  // same.
-  while (!done) {
-    bits_left = bs_remain(bs);
-    if (bits_left < 8) {
-      read = bs_peek(bs, bits_left) << (8 - bits_left);
-      done = 1;
-    } else {
-      read = bs_peek(bs, 8);
-      if (read == 0) {
-        bs_get(bs, 8);
-        bits += 8;
-      } else {
-        done = 1;
-      }
-    }
-  }
-  coded = exp_golomb_bits[read];
-  bs_get(bs, coded);
-  bits += coded;
-
-  //  printf("ue - bits %d\n", bits);
-  return bs_get(bs, bits + 1) - 1;
-}
-
-int32_t bs_se(struct bitstream* bs) {
-  uint32_t ret;
-  ret = bs_ue(bs);
-  if ((ret & 0x1) == 0) {
-    ret >>= 1;
-    int32_t temp = 0 - ret;
-    return temp;
-  }
-  return (ret + 1) >> 1;
-}

tools/lib/vidindex/bitstream.h

@@ -1,26 +0,0 @@
-#ifndef bitstream_H
-#define bitstream_H
-
-
-#include <stddef.h>
-#include <stdint.h>
-
-struct bitstream {
-  const uint8_t *buffer_ptr;
-  const uint8_t *buffer_end;
-  uint64_t value;
-  uint32_t pos;
-  uint32_t shift;
-  size_t size;
-};
-
-void bs_init(struct bitstream *bs, const uint8_t *buffer, size_t input_size);
-void bs_seek(struct bitstream *bs, size_t new_pos);
-uint32_t bs_get(struct bitstream *bs, int n);
-uint32_t bs_peek(struct bitstream *bs, int n);
-size_t bs_remain(struct bitstream *bs);
-int bs_eof(struct bitstream *bs);
-uint32_t bs_ue(struct bitstream *bs);
-int32_t bs_se(struct bitstream *bs);
-
-#endif

tools/lib/vidindex/vidindex.c

@@ -1,307 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdbool.h>
-#include <string.h>
-#include <assert.h>
-
-#include <unistd.h>
-#include <fcntl.h>
-#include <sys/stat.h>
-#include <sys/mman.h>
-
-#include "./bitstream.h"
-
-#define START_CODE 0x000001
-
-static uint32_t read24be(const uint8_t* ptr) {
-    return (ptr[0] << 16) | (ptr[1] << 8) | ptr[2];
-}
-static void write32le(FILE *of, uint32_t v) {
-  uint8_t va[4] = {
-    v & 0xff, (v >> 8) & 0xff, (v >> 16) & 0xff, (v >> 24) & 0xff
-  };
-  fwrite(va, 1, sizeof(va), of);
-}
-
-// Table 7-1
-enum hevc_nal_type {
-  HEVC_NAL_TYPE_TRAIL_N = 0,
-  HEVC_NAL_TYPE_TRAIL_R = 1,
-  HEVC_NAL_TYPE_TSA_N = 2,
-  HEVC_NAL_TYPE_TSA_R = 3,
-  HEVC_NAL_TYPE_STSA_N = 4,
-  HEVC_NAL_TYPE_STSA_R = 5,
-  HEVC_NAL_TYPE_RADL_N = 6,
-  HEVC_NAL_TYPE_RADL_R = 7,
-  HEVC_NAL_TYPE_RASL_N = 8,
-  HEVC_NAL_TYPE_RASL_R = 9,
-  HEVC_NAL_TYPE_BLA_W_LP = 16,
-  HEVC_NAL_TYPE_BLA_W_RADL = 17,
-  HEVC_NAL_TYPE_BLA_N_LP = 18,
-  HEVC_NAL_TYPE_IDR_W_RADL = 19,
-  HEVC_NAL_TYPE_IDR_N_LP = 20,
-  HEVC_NAL_TYPE_CRA_NUT = 21,
-  HEVC_NAL_TYPE_RSV_IRAP_VCL23 = 23,
-  HEVC_NAL_TYPE_VPS_NUT = 32,
-  HEVC_NAL_TYPE_SPS_NUT = 33,
-  HEVC_NAL_TYPE_PPS_NUT = 34,
-  HEVC_NAL_TYPE_AUD_NUT = 35,
-  HEVC_NAL_TYPE_EOS_NUT = 36,
-  HEVC_NAL_TYPE_EOB_NUT = 37,
-  HEVC_NAL_TYPE_FD_NUT = 38,
-  HEVC_NAL_TYPE_PREFIX_SEI_NUT = 39,
-  HEVC_NAL_TYPE_SUFFIX_SEI_NUT = 40,
-};
-
-// Table 7-7
-enum hevc_slice_type {
-  HEVC_SLICE_B = 0,
-  HEVC_SLICE_P = 1,
-  HEVC_SLICE_I = 2,
-};
-
-static void hevc_index(const uint8_t *data, size_t file_size, FILE *of_prefix, FILE *of_index) {
-  const uint8_t* ptr = data;
-  const uint8_t* ptr_end = data + file_size;
-
-  assert(ptr[0] == 0);
-  ptr++;
-  assert(read24be(ptr) == START_CODE);
-
-  // pps. ignore for now
-  uint32_t num_extra_slice_header_bits = 0;
-  uint32_t dependent_slice_segments_enabled_flag = 0;
-
-  while (ptr < ptr_end) {
-    const uint8_t* next = ptr+1;
-    for (; next < ptr_end-4; next++) {
-      if (read24be(next) == START_CODE) break;
-    }
-    size_t nal_size = next - ptr;
-    if (nal_size < 6) {
-      break;
-    }
-
-    {
-      struct bitstream bs = {0};
-      bs_init(&bs, ptr, nal_size);
-
-      uint32_t start_code = bs_get(&bs, 24);
-      assert(start_code == 0x000001);
-
-      // nal_unit_header
-      uint32_t forbidden_zero_bit = bs_get(&bs, 1);
-      uint32_t nal_unit_type = bs_get(&bs, 6);
-      uint32_t nuh_layer_id = bs_get(&bs, 6);
-      uint32_t nuh_temporal_id_plus1 = bs_get(&bs, 3);
-
-      // if (nal_unit_type != 1) printf("%3d -- %3d %10d %lu\n", nal_unit_type, frame_num, (uint32_t)(ptr-data), nal_size);
-
-      switch (nal_unit_type) {
-      case HEVC_NAL_TYPE_VPS_NUT:
-      case HEVC_NAL_TYPE_SPS_NUT:
-      case HEVC_NAL_TYPE_PPS_NUT:
-        fwrite(ptr, 1, nal_size, of_prefix);
-        break;
-      case HEVC_NAL_TYPE_TRAIL_N:
-      case HEVC_NAL_TYPE_TRAIL_R:
-      case HEVC_NAL_TYPE_TSA_N:
-      case HEVC_NAL_TYPE_TSA_R:
-      case HEVC_NAL_TYPE_STSA_N:
-      case HEVC_NAL_TYPE_STSA_R:
-      case HEVC_NAL_TYPE_RADL_N:
-      case HEVC_NAL_TYPE_RADL_R:
-      case HEVC_NAL_TYPE_RASL_N:
-      case HEVC_NAL_TYPE_RASL_R:
-      case HEVC_NAL_TYPE_BLA_W_LP:
-      case HEVC_NAL_TYPE_BLA_W_RADL:
-      case HEVC_NAL_TYPE_BLA_N_LP:
-      case HEVC_NAL_TYPE_IDR_W_RADL:
-      case HEVC_NAL_TYPE_IDR_N_LP:
-      case HEVC_NAL_TYPE_CRA_NUT: {
-        // slice_segment_header
-        uint32_t first_slice_segment_in_pic_flag = bs_get(&bs, 1);
-        if (nal_unit_type >= HEVC_NAL_TYPE_BLA_W_LP && nal_unit_type <= HEVC_NAL_TYPE_RSV_IRAP_VCL23) {
-          uint32_t no_output_of_prior_pics_flag = bs_get(&bs, 1);
-        }
-        uint32_t slice_pic_parameter_set_id = bs_get(&bs, 1);
-        if (!first_slice_segment_in_pic_flag) {
-          // ...
-          break;
-        }
-
-        if (!dependent_slice_segments_enabled_flag) {
-          for (int i=0; i<num_extra_slice_header_bits; i++) {
-            bs_get(&bs, 1);
-          }
-          uint32_t slice_type = bs_ue(&bs);
-
-          // write the index
-          write32le(of_index, slice_type);
-          write32le(of_index, ptr - data);
-
-          // ...
-        }
-
-        break;
-      }
-      }
-
-      //...
-      // emulation_prevention_three_byte
-    }
-
-    ptr = next;
-  }
-
-  write32le(of_index, -1);
-  write32le(of_index, file_size);
-}
-
-// Table 7-1
-enum h264_nal_type {
-  H264_NAL_SLICE           = 1,
-  H264_NAL_DPA             = 2,
-  H264_NAL_DPB             = 3,
-  H264_NAL_DPC             = 4,
-  H264_NAL_IDR_SLICE       = 5,
-  H264_NAL_SEI             = 6,
-  H264_NAL_SPS             = 7,
-  H264_NAL_PPS             = 8,
-  H264_NAL_AUD             = 9,
-  H264_NAL_END_SEQUENCE    = 10,
-  H264_NAL_END_STREAM      = 11,
-  H264_NAL_FILLER_DATA     = 12,
-  H264_NAL_SPS_EXT         = 13,
-  H264_NAL_AUXILIARY_SLICE = 19,
-};
-
-enum h264_slice_type {
-  H264_SLICE_P = 0,
-  H264_SLICE_B = 1,
-  H264_SLICE_I = 2,
-  // ...
-};
-
-static void h264_index(const uint8_t *data, size_t file_size, FILE *of_prefix, FILE *of_index) {
-  const uint8_t* ptr = data;
-  const uint8_t* ptr_end = data + file_size;
-
-  assert(ptr[0] == 0);
-  ptr++;
-  assert(read24be(ptr) == START_CODE);
-
-
-  uint32_t sps_log2_max_frame_num_minus4;
-
-
-  int last_frame_num = -1;
-
-  while (ptr < ptr_end) {
-    const uint8_t* next = ptr+1;
-    for (; next < ptr_end-4; next++) {
-      if (read24be(next) == START_CODE) break;
-    }
-    size_t nal_size = next - ptr;
-    if (nal_size < 5) {
-      break;
-    }
-
-    {
-      struct bitstream bs = {0};
-      bs_init(&bs, ptr, nal_size);
-
-      uint32_t start_code = bs_get(&bs, 24);
-      assert(start_code == 0x000001);
-
-      // nal_unit_header
-      uint32_t forbidden_zero_bit = bs_get(&bs, 1);
-      uint32_t nal_ref_idx = bs_get(&bs, 2);
-      uint32_t nal_unit_type = bs_get(&bs, 5);
-
-      switch (nal_unit_type) {
-      case H264_NAL_SPS:
-
-        {
-          uint32_t profile_idx = bs_get(&bs, 8);
-          uint32_t constraint_sets = bs_get(&bs, 4);
-          uint32_t reserved = bs_get(&bs, 5);
-          uint32_t level_idc = bs_get(&bs, 5);
-          uint32_t seq_parameter_set_id = bs_ue(&bs);
-          sps_log2_max_frame_num_minus4 = bs_ue(&bs);
-        }
-
-        // fallthrough
-      case H264_NAL_PPS:
-        fwrite(ptr, 1, nal_size, of_prefix);
-        break;
-
-      case H264_NAL_SLICE:
-      case H264_NAL_IDR_SLICE: {
-        // slice header
-        uint32_t first_mb_in_slice = bs_ue(&bs);
-        uint32_t slice_type = bs_ue(&bs);
-        uint32_t pic_parameter_set_id = bs_ue(&bs);
-
-        uint32_t frame_num = bs_get(&bs, sps_log2_max_frame_num_minus4+4);
-
-        if (first_mb_in_slice == 0) {
-          write32le(of_index, slice_type);
-          write32le(of_index, ptr - data);
-        }
-
-        break;
-      }
-
-      }
-
-    }
-
-    ptr = next;
-  }
-
-  write32le(of_index, -1);
-  write32le(of_index, file_size);
-}
-
-int main(int argc, char** argv) {
-  if (argc != 5) {
-    fprintf(stderr, "usage: %s h264|hevc file_path out_prefix out_index\n", argv[0]);
-    exit(1);
-  }
-
-  const char* file_type = argv[1];
-  const char* file_path = argv[2];
-
-  int fd = open(file_path, O_RDONLY, 0);
-  if (fd < 0) {
-    fprintf(stderr, "error: couldn't open %s\n", file_path);
-    exit(1);
-  }
-
-  FILE *of_prefix = fopen(argv[3], "wb");
-  assert(of_prefix);
-  FILE *of_index = fopen(argv[4], "wb");
-  assert(of_index);
-
-  off_t file_size = lseek(fd, 0, SEEK_END);
-  lseek(fd, 0, SEEK_SET);
-
-  assert(file_size > 4);
-
-  const uint8_t* data = (const uint8_t*)mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0);
-  assert(data != MAP_FAILED);
-
-  if (strcmp(file_type, "hevc") == 0) {
-    hevc_index(data, file_size, of_prefix, of_index);
-  } else if (strcmp(file_type, "h264") == 0) {
-    h264_index(data, file_size, of_prefix, of_index);
-  } else {
-    assert(false);
-  }
-
-  munmap((void*)data, file_size);
-  close(fd);
-
-  return 0;
-}

tools/replay/replay.cc

@@ -433,7 +433,7 @@ void Replay::stream() {
         long etime = (cur_mono_time_ - evt_start_ts) / speed_;
         long rtime = nanos_since_boot() - loop_start_ts;
         long behind_ns = etime - rtime;
-        // if behind_ns is greater than 1 second, it means that an invalid segemnt is skipped by seeking/replaying
+        // if behind_ns is greater than 1 second, it means that an invalid segment is skipped by seeking/replaying
         if (behind_ns >= 1 * 1e9 || speed_ != prev_replay_speed) {
           // reset event start times
           evt_start_ts = cur_mono_time_;

tools/sim/README.md

@@ -57,7 +57,13 @@ Start bridge processes located in tools/sim:
 
 ## Carla
 
-CARLA is also partially supported, though the performance is not great. openpilot doesn't have any extreme hardware requirements, however CARLA requires an NVIDIA graphics card and is very resource-intensive and may not run smoothly on your system.
+CARLA is also partially supported, though the performance is not great. It needs to be manually installed with:
+
+```bash
+poetry install --with=carla
+```
+
+openpilot doesn't have any extreme hardware requirements, however CARLA requires an NVIDIA graphics card and is very resource-intensive and may not run smoothly on your system.
 For this case, we have the simulator in low quality by default.
 
 You can also check out the [CARLA python documentation](https://carla.readthedocs.io/en/latest/python_api/) to find more parameters to tune that might increase performance on your system.

tools/sim/bridge/carla/carla_bridge.py

@@ -1,5 +1,3 @@
-import carla
-
 from openpilot.tools.sim.bridge.common import SimulatorBridge
 from openpilot.tools.sim.bridge.carla.carla_world import CarlaWorld
 
@@ -15,6 +13,8 @@ class CarlaBridge(SimulatorBridge):
     self.num_selected_spawn_point = arguments.num_selected_spawn_point
 
   def spawn_world(self):
+    import carla
+
     client = carla.Client(self.host, self.port)
     client.set_timeout(5)
 

tools/sim/bridge/carla/carla_world.py

@@ -1,4 +1,3 @@
-import carla
 import numpy as np
 
 from openpilot.common.params import Params
@@ -10,6 +9,8 @@ from openpilot.tools.sim.lib.camerad import W, H
 class CarlaWorld(World):
   def __init__(self, client, high_quality, dual_camera, num_selected_spawn_point, town):
     super().__init__(dual_camera)
+    import carla
+
     low_quality_layers = carla.MapLayer(carla.MapLayer.Ground | carla.MapLayer.Walls | carla.MapLayer.Decals)
 
     layers = carla.MapLayer.All if high_quality else low_quality_layers
@@ -139,5 +140,6 @@ class CarlaWorld(World):
     self.world.tick()
 
   def reset(self):
+    import carla
     self.vehicle.set_transform(self.spawn_point)
     self.vehicle.set_target_velocity(carla.Vector3D())

tools/sim/bridge/metadrive/metadrive_bridge.py

@@ -112,7 +112,8 @@ class MetaDriveBridge(SimulatorBridge):
         ]
       ),
       decision_repeat=1,
-      physics_world_step_size=self.TICKS_PER_FRAME/100
+      physics_world_step_size=self.TICKS_PER_FRAME/100,
+      preload_models=False
     )
 
     return MetaDriveWorld(config)

tools/sim/bridge/metadrive/metadrive_process.py

@@ -45,7 +45,12 @@ def metadrive_process(dual_camera: bool, config: dict, camera_array, controls_re
   road_image = np.frombuffer(camera_array.get_obj(), dtype=np.uint8).reshape((H, W, 3))
 
   env = MetaDriveEnv(config)
+
+  def reset():
     env.reset()
+    env.vehicle.config["max_speed_km_h"] = 1000
+
+  reset()
 
   def get_cam_as_rgb(cam):
     cam = env.engine.sensors[cam]
@@ -71,21 +76,21 @@ def metadrive_process(dual_camera: bool, config: dict, camera_array, controls_re
 
     if controls_recv.poll(0):
       while controls_recv.poll(0):
-        steer_angle, gas, reset = controls_recv.recv()
+        steer_angle, gas, should_reset = controls_recv.recv()
 
       steer_metadrive = steer_angle * 1 / (env.vehicle.MAX_STEERING * steer_ratio)
       steer_metadrive = np.clip(steer_metadrive, -1, 1)
 
       vc = [steer_metadrive, gas]
 
-      if reset:
+      if should_reset:
-        env.reset()
+        reset()
 
     if rk.frame % 5 == 0:
       obs, _, terminated, _, info = env.step(vc)
 
       if terminated:
-        env.reset()
+        reset()
 
       #if dual_camera:
       #  wide_road_image = get_cam_as_rgb("rgb_wide")

tools/sim/bridge/metadrive/metadrive_world.py

@@ -38,11 +38,11 @@ class MetaDriveWorld(World):
     self.should_reset = False
 
   def apply_controls(self, steer_angle, throttle_out, brake_out):
-    if (time.monotonic() - self.reset_time) > 5:
+    if (time.monotonic() - self.reset_time) > 2:
       self.vc[0] = steer_angle
 
       if throttle_out:
-        self.vc[1] = throttle_out/10
+        self.vc[1] = throttle_out
       else:
         self.vc[1] = -brake_out
     else:
@@ -50,6 +50,7 @@ class MetaDriveWorld(World):
       self.vc[1] = 0
 
     self.controls_send.send([*self.vc, self.should_reset])
+    self.should_reset = False
 
   def read_sensors(self, state: SimulatorState):
     while self.state_recv.poll(0):

tools/sim/launch_openpilot.sh

@@ -12,6 +12,8 @@ if [[ "$CI" ]]; then
   export BLOCK="${BLOCK},ui"
 fi
 
+python -c "from openpilot.selfdrive.test.helpers import set_params_enabled; set_params_enabled()"
+
 SCRIPT_DIR=$(dirname "$0")
 OPENPILOT_DIR=$SCRIPT_DIR/../../