Merge remote-tracking branch 'origin/master' into nlp-driving

.github/workflows/selfdrive_tests.yaml

@@ -5,6 +5,7 @@ on:
     branches:
       - master
   pull_request:
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}

.pre-commit-config.yaml

@@ -41,7 +41,7 @@ repos:
         args: ['--explicit-package-bases']
         exclude: '^(third_party/)|(cereal/)|(opendbc/)|(panda/)|(laika/)|(laika_repo/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(xx/)'
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.0
+    rev: v0.1.1
     hooks:
     -   id: ruff
         exclude: '^(third_party/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(laika_repo/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)'

Jenkinsfile

@@ -64,7 +64,9 @@ def deviceStage(String stageName, String deviceType, List env, def steps) {
     docker.image('ghcr.io/commaai/alpine-ssh').inside('--user=root') {
       lock(resource: "", label: deviceType, inversePrecedence: true, variable: 'device_ip', quantity: 1) {
         timeout(time: 20, unit: 'MINUTES') {
-          device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"))
+          retry (3) {
+            device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"))
+          }
           steps.each { item ->
             device(device_ip, item[0], item[1])
           }

SConstruct

@@ -26,6 +26,10 @@ AddOption('--ubsan',
           action='store_true',
           help='turn on UBSan')
 
+AddOption('--coverage',
+          action='store_true',
+          help='build with test coverage options')
+
 AddOption('--clazy',
           action='store_true',
           help='build with clazy')

common/prefix.py

@@ -14,6 +14,7 @@ class OpenpilotPrefix:
     self.clean_dirs_on_exit = clean_dirs_on_exit
 
   def __enter__(self):
+    self.original_prefix = os.environ.get('OPENPILOT_PREFIX', None)
     os.environ['OPENPILOT_PREFIX'] = self.prefix
     try:
       os.mkdir(self.msgq_path)
@@ -28,6 +29,8 @@ class OpenpilotPrefix:
       self.clean_dirs()
     try:
       del os.environ['OPENPILOT_PREFIX']
+      if self.original_prefix is not None:
+        os.environ['OPENPILOT_PREFIX'] = self.original_prefix
     except KeyError:
       pass
     return False

conftest.py

@@ -10,8 +10,13 @@ def openpilot_function_fixture():
 
   # setup a clean environment for each test
   with OpenpilotPrefix():
+    prefix = os.environ["OPENPILOT_PREFIX"]
+
     yield
 
+    # ensure the test doesn't change the prefix
+    assert "OPENPILOT_PREFIX" in os.environ and prefix == os.environ["OPENPILOT_PREFIX"]
+
   os.environ.clear()
   os.environ.update(starting_env)
 

docs/CARS.md

@@ -55,9 +55,9 @@ A supported vehicle is one that just works when you install a comma device. All
 |Honda|Accord Hybrid 2018-22|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|3 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Bosch A connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=Accord Hybrid 2018-22">Buy Here</a></sub></details>||
 |Honda|Civic 2016-18|Honda Sensing|openpilot|0 mph|12 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Nidec connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=Civic 2016-18">Buy Here</a></sub></details>|<a href="https://youtu.be/-IkImTe1NYE" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Honda|Civic 2019-21|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|2 mph[<sup>5</sup>](#footnotes)|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Bosch A connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=Civic 2019-21">Buy Here</a></sub></details>|<a href="https://www.youtube.com/watch?v=4Iz1Mz5LGF8" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
-|Honda|Civic 2022|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Bosch B connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=Civic 2022">Buy Here</a></sub></details>|<a href="https://youtu.be/ytiOT5lcp6Q" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
+|Honda|Civic 2022-23|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Bosch B connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=Civic 2022-23">Buy Here</a></sub></details>|<a href="https://youtu.be/ytiOT5lcp6Q" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Honda|Civic Hatchback 2017-21|Honda Sensing|openpilot available[<sup>1</sup>](#footnotes)|0 mph|12 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Bosch A connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=Civic Hatchback 2017-21">Buy Here</a></sub></details>||
-|Honda|Civic Hatchback 2022|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Bosch B connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=Civic Hatchback 2022">Buy Here</a></sub></details>|<a href="https://youtu.be/ytiOT5lcp6Q" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
+|Honda|Civic Hatchback 2022-23|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Bosch B connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=Civic Hatchback 2022-23">Buy Here</a></sub></details>|<a href="https://youtu.be/ytiOT5lcp6Q" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
 |Honda|CR-V 2015-16|Touring Trim|openpilot|25 mph|12 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Nidec connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=CR-V 2015-16">Buy Here</a></sub></details>||
 |Honda|CR-V 2017-22|Honda Sensing|openpilot available[<sup>1</sup>](#footnotes)|0 mph|12 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Bosch A connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=CR-V 2017-22">Buy Here</a></sub></details>||
 |Honda|CR-V Hybrid 2017-19|Honda Sensing|openpilot available[<sup>1</sup>](#footnotes)|0 mph|12 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Honda Bosch A connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Honda&model=CR-V Hybrid 2017-19">Buy Here</a></sub></details>||

panda

@@ -1 +1 @@
-Subproject commit 549fa32fc7b0354ebbb48bae846bff380eab9446
+Subproject commit f3bdfdd4354ccc3a512dc289dc038d5b30d1fec2

poetry.lock

@@ -314,13 +314,13 @@ files = [
 
 [[package]]
 name = "azure-core"
-version = "1.29.4"
+version = "1.29.5"
 description = "Microsoft Azure Core Library for Python"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "azure-core-1.29.4.tar.gz", hash = "sha256:500b3aa9bf2e90c5ccc88bb105d056114ca0ce7d0ce73afb8bc4d714b2fc7568"},
+    {file = "azure-core-1.29.5.tar.gz", hash = "sha256:52983c89d394c6f881a121e5101c5fa67278ca3b1f339c8fb2ef39230c70e9ac"},
-    {file = "azure_core-1.29.4-py3-none-any.whl", hash = "sha256:b03261bcba22c0b9290faf9999cedd23e849ed2577feee90515694cea6bc74bf"},
+    {file = "azure_core-1.29.5-py3-none-any.whl", hash = "sha256:0fa04b7b1f7d44a4fb8468c4093deb2ea01fdf4faddbf802ed9205615f99d68c"},
 ]
 
 [package.dependencies]
@@ -370,13 +370,13 @@ aio = ["azure-core[aio] (>=1.28.0,<2.0.0)"]
 
 [[package]]
 name = "babel"
-version = "2.13.0"
+version = "2.13.1"
 description = "Internationalization utilities"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "Babel-2.13.0-py3-none-any.whl", hash = "sha256:fbfcae1575ff78e26c7449136f1abbefc3c13ce542eeb13d43d50d8b047216ec"},
+    {file = "Babel-2.13.1-py3-none-any.whl", hash = "sha256:7077a4984b02b6727ac10f1f7294484f737443d7e2e66c5e4380e41a3ae0b4ed"},
-    {file = "Babel-2.13.0.tar.gz", hash = "sha256:04c3e2d28d2b7681644508f836be388ae49e0cfe91465095340395b60d00f210"},
+    {file = "Babel-2.13.1.tar.gz", hash = "sha256:33e0952d7dd6374af8dbf6768cc4ddf3ccfefc244f9986d4074704f2fbd18900"},
 ]
 
 [package.extras]
@@ -554,101 +554,101 @@ files = [
 
 [[package]]
 name = "charset-normalizer"
-version = "3.3.0"
+version = "3.3.1"
 description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
 optional = false
 python-versions = ">=3.7.0"
 files = [
-    {file = "charset-normalizer-3.3.0.tar.gz", hash = "sha256:63563193aec44bce707e0c5ca64ff69fa72ed7cf34ce6e11d5127555756fd2f6"},
+    {file = "charset-normalizer-3.3.1.tar.gz", hash = "sha256:d9137a876020661972ca6eec0766d81aef8a5627df628b664b234b73396e727e"},
-    {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:effe5406c9bd748a871dbcaf3ac69167c38d72db8c9baf3ff954c344f31c4cbe"},
+    {file = "charset_normalizer-3.3.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8aee051c89e13565c6bd366813c386939f8e928af93c29fda4af86d25b73d8f8"},
-    {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4162918ef3098851fcd8a628bf9b6a98d10c380725df9e04caf5ca6dd48c847a"},
+    {file = "charset_normalizer-3.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:352a88c3df0d1fa886562384b86f9a9e27563d4704ee0e9d56ec6fcd270ea690"},
-    {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0570d21da019941634a531444364f2482e8db0b3425fcd5ac0c36565a64142c8"},
+    {file = "charset_normalizer-3.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:223b4d54561c01048f657fa6ce41461d5ad8ff128b9678cfe8b2ecd951e3f8a2"},
-    {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5707a746c6083a3a74b46b3a631d78d129edab06195a92a8ece755aac25a3f3d"},
+    {file = "charset_normalizer-3.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f861d94c2a450b974b86093c6c027888627b8082f1299dfd5a4bae8e2292821"},
-    {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:278c296c6f96fa686d74eb449ea1697f3c03dc28b75f873b65b5201806346a69"},
+    {file = "charset_normalizer-3.3.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1171ef1fc5ab4693c5d151ae0fdad7f7349920eabbaca6271f95969fa0756c2d"},
-    {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a4b71f4d1765639372a3b32d2638197f5cd5221b19531f9245fcc9ee62d38f56"},
+    {file = "charset_normalizer-3.3.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28f512b9a33235545fbbdac6a330a510b63be278a50071a336afc1b78781b147"},
-    {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5969baeaea61c97efa706b9b107dcba02784b1601c74ac84f2a532ea079403e"},
+    {file = "charset_normalizer-3.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0e842112fe3f1a4ffcf64b06dc4c61a88441c2f02f373367f7b4c1aa9be2ad5"},
-    {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3f93dab657839dfa61025056606600a11d0b696d79386f974e459a3fbc568ec"},
+    {file = "charset_normalizer-3.3.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f9bc2ce123637a60ebe819f9fccc614da1bcc05798bbbaf2dd4ec91f3e08846"},
-    {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:db756e48f9c5c607b5e33dd36b1d5872d0422e960145b08ab0ec7fd420e9d649"},
+    {file = "charset_normalizer-3.3.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:f194cce575e59ffe442c10a360182a986535fd90b57f7debfaa5c845c409ecc3"},
-    {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:232ac332403e37e4a03d209a3f92ed9071f7d3dbda70e2a5e9cff1c4ba9f0678"},
+    {file = "charset_normalizer-3.3.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9a74041ba0bfa9bc9b9bb2cd3238a6ab3b7618e759b41bd15b5f6ad958d17605"},
-    {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e5c1502d4ace69a179305abb3f0bb6141cbe4714bc9b31d427329a95acfc8bdd"},
+    {file = "charset_normalizer-3.3.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b578cbe580e3b41ad17b1c428f382c814b32a6ce90f2d8e39e2e635d49e498d1"},
-    {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:2502dd2a736c879c0f0d3e2161e74d9907231e25d35794584b1ca5284e43f596"},
+    {file = "charset_normalizer-3.3.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:6db3cfb9b4fcecb4390db154e75b49578c87a3b9979b40cdf90d7e4b945656e1"},
-    {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23e8565ab7ff33218530bc817922fae827420f143479b753104ab801145b1d5b"},
+    {file = "charset_normalizer-3.3.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:debb633f3f7856f95ad957d9b9c781f8e2c6303ef21724ec94bea2ce2fcbd056"},
-    {file = "charset_normalizer-3.3.0-cp310-cp310-win32.whl", hash = "sha256:1872d01ac8c618a8da634e232f24793883d6e456a66593135aeafe3784b0848d"},
+    {file = "charset_normalizer-3.3.1-cp310-cp310-win32.whl", hash = "sha256:87071618d3d8ec8b186d53cb6e66955ef2a0e4fa63ccd3709c0c90ac5a43520f"},
-    {file = "charset_normalizer-3.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:557b21a44ceac6c6b9773bc65aa1b4cc3e248a5ad2f5b914b91579a32e22204d"},
+    {file = "charset_normalizer-3.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:e372d7dfd154009142631de2d316adad3cc1c36c32a38b16a4751ba78da2a397"},
-    {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d7eff0f27edc5afa9e405f7165f85a6d782d308f3b6b9d96016c010597958e63"},
+    {file = "charset_normalizer-3.3.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ae4070f741f8d809075ef697877fd350ecf0b7c5837ed68738607ee0a2c572cf"},
-    {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6a685067d05e46641d5d1623d7c7fdf15a357546cbb2f71b0ebde91b175ffc3e"},
+    {file = "charset_normalizer-3.3.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:58e875eb7016fd014c0eea46c6fa92b87b62c0cb31b9feae25cbbe62c919f54d"},
-    {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0d3d5b7db9ed8a2b11a774db2bbea7ba1884430a205dbd54a32d61d7c2a190fa"},
+    {file = "charset_normalizer-3.3.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dbd95e300367aa0827496fe75a1766d198d34385a58f97683fe6e07f89ca3e3c"},
-    {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2935ffc78db9645cb2086c2f8f4cfd23d9b73cc0dc80334bc30aac6f03f68f8c"},
+    {file = "charset_normalizer-3.3.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de0b4caa1c8a21394e8ce971997614a17648f94e1cd0640fbd6b4d14cab13a72"},
-    {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fe359b2e3a7729010060fbca442ca225280c16e923b37db0e955ac2a2b72a05"},
+    {file = "charset_normalizer-3.3.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:985c7965f62f6f32bf432e2681173db41336a9c2611693247069288bcb0c7f8b"},
-    {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:380c4bde80bce25c6e4f77b19386f5ec9db230df9f2f2ac1e5ad7af2caa70459"},
+    {file = "charset_normalizer-3.3.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a15c1fe6d26e83fd2e5972425a772cca158eae58b05d4a25a4e474c221053e2d"},
-    {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0d1e3732768fecb052d90d62b220af62ead5748ac51ef61e7b32c266cac9293"},
+    {file = "charset_normalizer-3.3.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae55d592b02c4349525b6ed8f74c692509e5adffa842e582c0f861751701a673"},
-    {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1b2919306936ac6efb3aed1fbf81039f7087ddadb3160882a57ee2ff74fd2382"},
+    {file = "charset_normalizer-3.3.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:be4d9c2770044a59715eb57c1144dedea7c5d5ae80c68fb9959515037cde2008"},
-    {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f8888e31e3a85943743f8fc15e71536bda1c81d5aa36d014a3c0c44481d7db6e"},
+    {file = "charset_normalizer-3.3.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:851cf693fb3aaef71031237cd68699dded198657ec1e76a76eb8be58c03a5d1f"},
-    {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:82eb849f085624f6a607538ee7b83a6d8126df6d2f7d3b319cb837b289123078"},
+    {file = "charset_normalizer-3.3.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:31bbaba7218904d2eabecf4feec0d07469284e952a27400f23b6628439439fa7"},
-    {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7b8b8bf1189b3ba9b8de5c8db4d541b406611a71a955bbbd7385bbc45fcb786c"},
+    {file = "charset_normalizer-3.3.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:871d045d6ccc181fd863a3cd66ee8e395523ebfbc57f85f91f035f50cee8e3d4"},
-    {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5adf257bd58c1b8632046bbe43ee38c04e1038e9d37de9c57a94d6bd6ce5da34"},
+    {file = "charset_normalizer-3.3.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:501adc5eb6cd5f40a6f77fbd90e5ab915c8fd6e8c614af2db5561e16c600d6f3"},
-    {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c350354efb159b8767a6244c166f66e67506e06c8924ed74669b2c70bc8735b1"},
+    {file = "charset_normalizer-3.3.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f5fb672c396d826ca16a022ac04c9dce74e00a1c344f6ad1a0fdc1ba1f332213"},
-    {file = "charset_normalizer-3.3.0-cp311-cp311-win32.whl", hash = "sha256:02af06682e3590ab952599fbadac535ede5d60d78848e555aa58d0c0abbde786"},
+    {file = "charset_normalizer-3.3.1-cp311-cp311-win32.whl", hash = "sha256:bb06098d019766ca16fc915ecaa455c1f1cd594204e7f840cd6258237b5079a8"},
-    {file = "charset_normalizer-3.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:86d1f65ac145e2c9ed71d8ffb1905e9bba3a91ae29ba55b4c46ae6fc31d7c0d4"},
+    {file = "charset_normalizer-3.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:8af5a8917b8af42295e86b64903156b4f110a30dca5f3b5aedea123fbd638bff"},
-    {file = "charset_normalizer-3.3.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:3b447982ad46348c02cb90d230b75ac34e9886273df3a93eec0539308a6296d7"},
+    {file = "charset_normalizer-3.3.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:7ae8e5142dcc7a49168f4055255dbcced01dc1714a90a21f87448dc8d90617d1"},
-    {file = "charset_normalizer-3.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:abf0d9f45ea5fb95051c8bfe43cb40cda383772f7e5023a83cc481ca2604d74e"},
+    {file = "charset_normalizer-3.3.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5b70bab78accbc672f50e878a5b73ca692f45f5b5e25c8066d748c09405e6a55"},
-    {file = "charset_normalizer-3.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b09719a17a2301178fac4470d54b1680b18a5048b481cb8890e1ef820cb80455"},
+    {file = "charset_normalizer-3.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5ceca5876032362ae73b83347be8b5dbd2d1faf3358deb38c9c88776779b2e2f"},
-    {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3d9b48ee6e3967b7901c052b670c7dda6deb812c309439adaffdec55c6d7b78"},
+    {file = "charset_normalizer-3.3.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34d95638ff3613849f473afc33f65c401a89f3b9528d0d213c7037c398a51296"},
-    {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:edfe077ab09442d4ef3c52cb1f9dab89bff02f4524afc0acf2d46be17dc479f5"},
+    {file = "charset_normalizer-3.3.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9edbe6a5bf8b56a4a84533ba2b2f489d0046e755c29616ef8830f9e7d9cf5728"},
-    {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3debd1150027933210c2fc321527c2299118aa929c2f5a0a80ab6953e3bd1908"},
+    {file = "charset_normalizer-3.3.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6a02a3c7950cafaadcd46a226ad9e12fc9744652cc69f9e5534f98b47f3bbcf"},
-    {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86f63face3a527284f7bb8a9d4f78988e3c06823f7bea2bd6f0e0e9298ca0403"},
+    {file = "charset_normalizer-3.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10b8dd31e10f32410751b3430996f9807fc4d1587ca69772e2aa940a82ab571a"},
-    {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:24817cb02cbef7cd499f7c9a2735286b4782bd47a5b3516a0e84c50eab44b98e"},
+    {file = "charset_normalizer-3.3.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edc0202099ea1d82844316604e17d2b175044f9bcb6b398aab781eba957224bd"},
-    {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c71f16da1ed8949774ef79f4a0260d28b83b3a50c6576f8f4f0288d109777989"},
+    {file = "charset_normalizer-3.3.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b891a2f68e09c5ef989007fac11476ed33c5c9994449a4e2c3386529d703dc8b"},
-    {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:9cf3126b85822c4e53aa28c7ec9869b924d6fcfb76e77a45c44b83d91afd74f9"},
+    {file = "charset_normalizer-3.3.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:71ef3b9be10070360f289aea4838c784f8b851be3ba58cf796262b57775c2f14"},
-    {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:b3b2316b25644b23b54a6f6401074cebcecd1244c0b8e80111c9a3f1c8e83d65"},
+    {file = "charset_normalizer-3.3.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:55602981b2dbf8184c098bc10287e8c245e351cd4fdcad050bd7199d5a8bf514"},
-    {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:03680bb39035fbcffe828eae9c3f8afc0428c91d38e7d61aa992ef7a59fb120e"},
+    {file = "charset_normalizer-3.3.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:46fb9970aa5eeca547d7aa0de5d4b124a288b42eaefac677bde805013c95725c"},
-    {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4cc152c5dd831641e995764f9f0b6589519f6f5123258ccaca8c6d34572fefa8"},
+    {file = "charset_normalizer-3.3.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:520b7a142d2524f999447b3a0cf95115df81c4f33003c51a6ab637cbda9d0bf4"},
-    {file = "charset_normalizer-3.3.0-cp312-cp312-win32.whl", hash = "sha256:b8f3307af845803fb0b060ab76cf6dd3a13adc15b6b451f54281d25911eb92df"},
+    {file = "charset_normalizer-3.3.1-cp312-cp312-win32.whl", hash = "sha256:8ec8ef42c6cd5856a7613dcd1eaf21e5573b2185263d87d27c8edcae33b62a61"},
-    {file = "charset_normalizer-3.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:8eaf82f0eccd1505cf39a45a6bd0a8cf1c70dcfc30dba338207a969d91b965c0"},
+    {file = "charset_normalizer-3.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:baec8148d6b8bd5cee1ae138ba658c71f5b03e0d69d5907703e3e1df96db5e41"},
-    {file = "charset_normalizer-3.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:dc45229747b67ffc441b3de2f3ae5e62877a282ea828a5bdb67883c4ee4a8810"},
+    {file = "charset_normalizer-3.3.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63a6f59e2d01310f754c270e4a257426fe5a591dc487f1983b3bbe793cf6bac6"},
-    {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f4a0033ce9a76e391542c182f0d48d084855b5fcba5010f707c8e8c34663d77"},
+    {file = "charset_normalizer-3.3.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d6bfc32a68bc0933819cfdfe45f9abc3cae3877e1d90aac7259d57e6e0f85b1"},
-    {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ada214c6fa40f8d800e575de6b91a40d0548139e5dc457d2ebb61470abf50186"},
+    {file = "charset_normalizer-3.3.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4f3100d86dcd03c03f7e9c3fdb23d92e32abbca07e7c13ebd7ddfbcb06f5991f"},
-    {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b1121de0e9d6e6ca08289583d7491e7fcb18a439305b34a30b20d8215922d43c"},
+    {file = "charset_normalizer-3.3.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39b70a6f88eebe239fa775190796d55a33cfb6d36b9ffdd37843f7c4c1b5dc67"},
-    {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1063da2c85b95f2d1a430f1c33b55c9c17ffaf5e612e10aeaad641c55a9e2b9d"},
+    {file = "charset_normalizer-3.3.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e12f8ee80aa35e746230a2af83e81bd6b52daa92a8afaef4fea4a2ce9b9f4fa"},
-    {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:70f1d09c0d7748b73290b29219e854b3207aea922f839437870d8cc2168e31cc"},
+    {file = "charset_normalizer-3.3.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b6cefa579e1237ce198619b76eaa148b71894fb0d6bcf9024460f9bf30fd228"},
-    {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:250c9eb0f4600361dd80d46112213dff2286231d92d3e52af1e5a6083d10cad9"},
+    {file = "charset_normalizer-3.3.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:61f1e3fb621f5420523abb71f5771a204b33c21d31e7d9d86881b2cffe92c47c"},
-    {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:750b446b2ffce1739e8578576092179160f6d26bd5e23eb1789c4d64d5af7dc7"},
+    {file = "charset_normalizer-3.3.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4f6e2a839f83a6a76854d12dbebde50e4b1afa63e27761549d006fa53e9aa80e"},
-    {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:fc52b79d83a3fe3a360902d3f5d79073a993597d48114c29485e9431092905d8"},
+    {file = "charset_normalizer-3.3.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:1ec937546cad86d0dce5396748bf392bb7b62a9eeb8c66efac60e947697f0e58"},
-    {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:588245972aca710b5b68802c8cad9edaa98589b1b42ad2b53accd6910dad3545"},
+    {file = "charset_normalizer-3.3.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:82ca51ff0fc5b641a2d4e1cc8c5ff108699b7a56d7f3ad6f6da9dbb6f0145b48"},
-    {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e39c7eb31e3f5b1f88caff88bcff1b7f8334975b46f6ac6e9fc725d829bc35d4"},
+    {file = "charset_normalizer-3.3.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:633968254f8d421e70f91c6ebe71ed0ab140220469cf87a9857e21c16687c034"},
-    {file = "charset_normalizer-3.3.0-cp37-cp37m-win32.whl", hash = "sha256:abecce40dfebbfa6abf8e324e1860092eeca6f7375c8c4e655a8afb61af58f2c"},
+    {file = "charset_normalizer-3.3.1-cp37-cp37m-win32.whl", hash = "sha256:c0c72d34e7de5604df0fde3644cc079feee5e55464967d10b24b1de268deceb9"},
-    {file = "charset_normalizer-3.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:24a91a981f185721542a0b7c92e9054b7ab4fea0508a795846bc5b0abf8118d4"},
+    {file = "charset_normalizer-3.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:63accd11149c0f9a99e3bc095bbdb5a464862d77a7e309ad5938fbc8721235ae"},
-    {file = "charset_normalizer-3.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:67b8cc9574bb518ec76dc8e705d4c39ae78bb96237cb533edac149352c1f39fe"},
+    {file = "charset_normalizer-3.3.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5a3580a4fdc4ac05f9e53c57f965e3594b2f99796231380adb2baaab96e22761"},
-    {file = "charset_normalizer-3.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ac71b2977fb90c35d41c9453116e283fac47bb9096ad917b8819ca8b943abecd"},
+    {file = "charset_normalizer-3.3.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2465aa50c9299d615d757c1c888bc6fef384b7c4aec81c05a0172b4400f98557"},
-    {file = "charset_normalizer-3.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3ae38d325b512f63f8da31f826e6cb6c367336f95e418137286ba362925c877e"},
+    {file = "charset_normalizer-3.3.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cb7cd68814308aade9d0c93c5bd2ade9f9441666f8ba5aa9c2d4b389cb5e2a45"},
-    {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:542da1178c1c6af8873e143910e2269add130a299c9106eef2594e15dae5e482"},
+    {file = "charset_normalizer-3.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91e43805ccafa0a91831f9cd5443aa34528c0c3f2cc48c4cb3d9a7721053874b"},
-    {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30a85aed0b864ac88309b7d94be09f6046c834ef60762a8833b660139cfbad13"},
+    {file = "charset_normalizer-3.3.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:854cc74367180beb327ab9d00f964f6d91da06450b0855cbbb09187bcdb02de5"},
-    {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aae32c93e0f64469f74ccc730a7cb21c7610af3a775157e50bbd38f816536b38"},
+    {file = "charset_normalizer-3.3.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c15070ebf11b8b7fd1bfff7217e9324963c82dbdf6182ff7050519e350e7ad9f"},
-    {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15b26ddf78d57f1d143bdf32e820fd8935d36abe8a25eb9ec0b5a71c82eb3895"},
+    {file = "charset_normalizer-3.3.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c4c99f98fc3a1835af8179dcc9013f93594d0670e2fa80c83aa36346ee763d2"},
-    {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f5d10bae5d78e4551b7be7a9b29643a95aded9d0f602aa2ba584f0388e7a557"},
+    {file = "charset_normalizer-3.3.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3fb765362688821404ad6cf86772fc54993ec11577cd5a92ac44b4c2ba52155b"},
-    {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:249c6470a2b60935bafd1d1d13cd613f8cd8388d53461c67397ee6a0f5dce741"},
+    {file = "charset_normalizer-3.3.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:dced27917823df984fe0c80a5c4ad75cf58df0fbfae890bc08004cd3888922a2"},
-    {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c5a74c359b2d47d26cdbbc7845e9662d6b08a1e915eb015d044729e92e7050b7"},
+    {file = "charset_normalizer-3.3.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a66bcdf19c1a523e41b8e9d53d0cedbfbac2e93c649a2e9502cb26c014d0980c"},
-    {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:b5bcf60a228acae568e9911f410f9d9e0d43197d030ae5799e20dca8df588287"},
+    {file = "charset_normalizer-3.3.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:ecd26be9f112c4f96718290c10f4caea6cc798459a3a76636b817a0ed7874e42"},
-    {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:187d18082694a29005ba2944c882344b6748d5be69e3a89bf3cc9d878e548d5a"},
+    {file = "charset_normalizer-3.3.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:3f70fd716855cd3b855316b226a1ac8bdb3caf4f7ea96edcccc6f484217c9597"},
-    {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:81bf654678e575403736b85ba3a7867e31c2c30a69bc57fe88e3ace52fb17b89"},
+    {file = "charset_normalizer-3.3.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:17a866d61259c7de1bdadef418a37755050ddb4b922df8b356503234fff7932c"},
-    {file = "charset_normalizer-3.3.0-cp38-cp38-win32.whl", hash = "sha256:85a32721ddde63c9df9ebb0d2045b9691d9750cb139c161c80e500d210f5e26e"},
+    {file = "charset_normalizer-3.3.1-cp38-cp38-win32.whl", hash = "sha256:548eefad783ed787b38cb6f9a574bd8664468cc76d1538215d510a3cd41406cb"},
-    {file = "charset_normalizer-3.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:468d2a840567b13a590e67dd276c570f8de00ed767ecc611994c301d0f8c014f"},
+    {file = "charset_normalizer-3.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:45f053a0ece92c734d874861ffe6e3cc92150e32136dd59ab1fb070575189c97"},
-    {file = "charset_normalizer-3.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e0fc42822278451bc13a2e8626cf2218ba570f27856b536e00cfa53099724828"},
+    {file = "charset_normalizer-3.3.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bc791ec3fd0c4309a753f95bb6c749ef0d8ea3aea91f07ee1cf06b7b02118f2f"},
-    {file = "charset_normalizer-3.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:09c77f964f351a7369cc343911e0df63e762e42bac24cd7d18525961c81754f4"},
+    {file = "charset_normalizer-3.3.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0c8c61fb505c7dad1d251c284e712d4e0372cef3b067f7ddf82a7fa82e1e9a93"},
-    {file = "charset_normalizer-3.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:12ebea541c44fdc88ccb794a13fe861cc5e35d64ed689513a5c03d05b53b7c82"},
+    {file = "charset_normalizer-3.3.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2c092be3885a1b7899cd85ce24acedc1034199d6fca1483fa2c3a35c86e43041"},
-    {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:805dfea4ca10411a5296bcc75638017215a93ffb584c9e344731eef0dcfb026a"},
+    {file = "charset_normalizer-3.3.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c2000c54c395d9e5e44c99dc7c20a64dc371f777faf8bae4919ad3e99ce5253e"},
-    {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:96c2b49eb6a72c0e4991d62406e365d87067ca14c1a729a870d22354e6f68115"},
+    {file = "charset_normalizer-3.3.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4cb50a0335382aac15c31b61d8531bc9bb657cfd848b1d7158009472189f3d62"},
-    {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aaf7b34c5bc56b38c931a54f7952f1ff0ae77a2e82496583b247f7c969eb1479"},
+    {file = "charset_normalizer-3.3.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c30187840d36d0ba2893bc3271a36a517a717f9fd383a98e2697ee890a37c273"},
-    {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:619d1c96099be5823db34fe89e2582b336b5b074a7f47f819d6b3a57ff7bdb86"},
+    {file = "charset_normalizer-3.3.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe81b35c33772e56f4b6cf62cf4aedc1762ef7162a31e6ac7fe5e40d0149eb67"},
-    {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a0ac5e7015a5920cfce654c06618ec40c33e12801711da6b4258af59a8eff00a"},
+    {file = "charset_normalizer-3.3.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d0bf89afcbcf4d1bb2652f6580e5e55a840fdf87384f6063c4a4f0c95e378656"},
-    {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:93aa7eef6ee71c629b51ef873991d6911b906d7312c6e8e99790c0f33c576f89"},
+    {file = "charset_normalizer-3.3.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:06cf46bdff72f58645434d467bf5228080801298fbba19fe268a01b4534467f5"},
-    {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7966951325782121e67c81299a031f4c115615e68046f79b85856b86ebffc4cd"},
+    {file = "charset_normalizer-3.3.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:3c66df3f41abee950d6638adc7eac4730a306b022570f71dd0bd6ba53503ab57"},
-    {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:02673e456dc5ab13659f85196c534dc596d4ef260e4d86e856c3b2773ce09843"},
+    {file = "charset_normalizer-3.3.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:cd805513198304026bd379d1d516afbf6c3c13f4382134a2c526b8b854da1c2e"},
-    {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:c2af80fb58f0f24b3f3adcb9148e6203fa67dd3f61c4af146ecad033024dde43"},
+    {file = "charset_normalizer-3.3.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:9505dc359edb6a330efcd2be825fdb73ee3e628d9010597aa1aee5aa63442e97"},
-    {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:153e7b6e724761741e0974fc4dcd406d35ba70b92bfe3fedcb497226c93b9da7"},
+    {file = "charset_normalizer-3.3.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:31445f38053476a0c4e6d12b047b08ced81e2c7c712e5a1ad97bc913256f91b2"},
-    {file = "charset_normalizer-3.3.0-cp39-cp39-win32.whl", hash = "sha256:d47ecf253780c90ee181d4d871cd655a789da937454045b17b5798da9393901a"},
+    {file = "charset_normalizer-3.3.1-cp39-cp39-win32.whl", hash = "sha256:bd28b31730f0e982ace8663d108e01199098432a30a4c410d06fe08fdb9e93f4"},
-    {file = "charset_normalizer-3.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:d97d85fa63f315a8bdaba2af9a6a686e0eceab77b3089af45133252618e70884"},
+    {file = "charset_normalizer-3.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:555fe186da0068d3354cdf4bbcbc609b0ecae4d04c921cc13e209eece7720727"},
-    {file = "charset_normalizer-3.3.0-py3-none-any.whl", hash = "sha256:e46cd37076971c1040fc8c41273a8b3e2c624ce4f2be3f5dfcb7a430c1d3acc2"},
+    {file = "charset_normalizer-3.3.1-py3-none-any.whl", hash = "sha256:800561453acdecedaac137bf09cd719c7a440b6800ec182f077bb8e7025fb708"},
 ]
 
 [[package]]
@@ -3472,8 +3472,6 @@ files = [
     {file = "pygame-2.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e24d05184e4195fe5ebcdce8b18ecb086f00182b9ae460a86682d312ce8d31f"},
     {file = "pygame-2.5.2-cp311-cp311-win32.whl", hash = "sha256:f02c1c7505af18d426d355ac9872bd5c916b27f7b0fe224749930662bea47a50"},
     {file = "pygame-2.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:6d58c8cf937815d3b7cdc0fa9590c5129cb2c9658b72d00e8a4568dea2ff1d42"},
-    {file = "pygame-2.5.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1a2a43802bb5e89ce2b3b775744e78db4f9a201bf8d059b946c61722840ceea8"},
-    {file = "pygame-2.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1c289f2613c44fe70a1e40769de4a49c5ab5a29b9376f1692bb1a15c9c1c9bfa"},
     {file = "pygame-2.5.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:074aa6c6e110c925f7f27f00c7733c6303407edc61d738882985091d1eb2ef17"},
     {file = "pygame-2.5.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe0228501ec616779a0b9c4299e837877783e18df294dd690b9ab0eed3d8aaab"},
     {file = "pygame-2.5.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31648d38ecdc2335ffc0e38fb18a84b3339730521505dac68514f83a1092e3f4"},
@@ -3613,6 +3611,17 @@ files = [
 [package.dependencies]
 cffi = ">=1.0.0"
 
+[[package]]
+name = "pympler"
+version = "1.0.1"
+description = "A development tool to measure, monitor and analyze the memory behavior of Python objects."
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"},
+    {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"},
+]
+
 [[package]]
 name = "pyopencl"
 version = "2023.1.4"
@@ -4197,28 +4206,28 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
 
 [[package]]
 name = "ruff"
-version = "0.1.0"
+version = "0.1.1"
 description = "An extremely fast Python linter, written in Rust."
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "ruff-0.1.0-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:87114e254dee35e069e1b922d85d4b21a5b61aec759849f393e1dbb308a00439"},
+    {file = "ruff-0.1.1-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:b7cdc893aef23ccc14c54bd79a8109a82a2c527e11d030b62201d86f6c2b81c5"},
-    {file = "ruff-0.1.0-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:764f36d2982cc4a703e69fb73a280b7c539fd74b50c9ee531a4e3fe88152f521"},
+    {file = "ruff-0.1.1-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:620d4b34302538dbd8bbbe8fdb8e8f98d72d29bd47e972e2b59ce6c1e8862257"},
-    {file = "ruff-0.1.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65f4b7fb539e5cf0f71e9bd74f8ddab74cabdd673c6fb7f17a4dcfd29f126255"},
+    {file = "ruff-0.1.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a909d3930afdbc2e9fd893b0034479e90e7981791879aab50ce3d9f55205bd6"},
-    {file = "ruff-0.1.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:299fff467a0f163baa282266b310589b21400de0a42d8f68553422fa6bf7ee01"},
+    {file = "ruff-0.1.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3305d1cb4eb8ff6d3e63a48d1659d20aab43b49fe987b3ca4900528342367145"},
-    {file = "ruff-0.1.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0d412678bf205787263bb702c984012a4f97e460944c072fd7cfa2bd084857c4"},
+    {file = "ruff-0.1.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c34ae501d0ec71acf19ee5d4d889e379863dcc4b796bf8ce2934a9357dc31db7"},
-    {file = "ruff-0.1.0-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:a5391b49b1669b540924640587d8d24128e45be17d1a916b1801d6645e831581"},
+    {file = "ruff-0.1.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:6aa7e63c3852cf8fe62698aef31e563e97143a4b801b57f920012d0e07049a8d"},
-    {file = "ruff-0.1.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee8cd57f454cdd77bbcf1e11ff4e0046fb6547cac1922cc6e3583ce4b9c326d1"},
+    {file = "ruff-0.1.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2d68367d1379a6b47e61bc9de144a47bcdb1aad7903bbf256e4c3d31f11a87ae"},
-    {file = "ruff-0.1.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fa7aeed7bc23861a2b38319b636737bf11cfa55d2109620b49cf995663d3e888"},
+    {file = "ruff-0.1.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bc11955f6ce3398d2afe81ad7e49d0ebf0a581d8bcb27b8c300281737735e3a3"},
-    {file = "ruff-0.1.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b04cd4298b43b16824d9a37800e4c145ba75c29c43ce0d74cad1d66d7ae0a4c5"},
+    {file = "ruff-0.1.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cbbd8eead88ea83a250499074e2a8e9d80975f0b324b1e2e679e4594da318c25"},
-    {file = "ruff-0.1.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:7186ccf54707801d91e6314a016d1c7895e21d2e4cd614500d55870ed983aa9f"},
+    {file = "ruff-0.1.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f4780e2bb52f3863a565ec3f699319d3493b83ff95ebbb4993e59c62aaf6e75e"},
-    {file = "ruff-0.1.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d88adfd93849bc62449518228581d132e2023e30ebd2da097f73059900d8dce3"},
+    {file = "ruff-0.1.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8f5b24daddf35b6c207619301170cae5d2699955829cda77b6ce1e5fc69340df"},
-    {file = "ruff-0.1.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:ad2ccdb3bad5a61013c76a9c1240fdfadf2c7103a2aeebd7bcbbed61f363138f"},
+    {file = "ruff-0.1.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d3f9ac658ba29e07b95c80fa742b059a55aefffa8b1e078bc3c08768bdd4b11a"},
-    {file = "ruff-0.1.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b77f6cfa72c6eb19b5cac967cc49762ae14d036db033f7d97a72912770fd8e1c"},
+    {file = "ruff-0.1.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:3521bf910104bf781e6753282282acc145cbe3eff79a1ce6b920404cd756075a"},
-    {file = "ruff-0.1.0-py3-none-win32.whl", hash = "sha256:480bd704e8af1afe3fd444cc52e3c900b936e6ca0baf4fb0281124330b6ceba2"},
+    {file = "ruff-0.1.1-py3-none-win32.whl", hash = "sha256:ba3208543ab91d3e4032db2652dcb6c22a25787b85b8dc3aeff084afdc612e5c"},
-    {file = "ruff-0.1.0-py3-none-win_amd64.whl", hash = "sha256:a76ba81860f7ee1f2d5651983f87beb835def94425022dc5f0803108f1b8bfa2"},
+    {file = "ruff-0.1.1-py3-none-win_amd64.whl", hash = "sha256:3ff3006c97d9dc396b87fb46bb65818e614ad0181f059322df82bbfe6944e264"},
-    {file = "ruff-0.1.0-py3-none-win_arm64.whl", hash = "sha256:45abdbdab22509a2c6052ecf7050b3f5c7d6b7898dc07e82869401b531d46da4"},
+    {file = "ruff-0.1.1-py3-none-win_arm64.whl", hash = "sha256:e140bd717c49164c8feb4f65c644046fe929c46f42493672853e3213d7bdbce2"},
-    {file = "ruff-0.1.0.tar.gz", hash = "sha256:ad6b13824714b19c5f8225871cf532afb994470eecb74631cd3500fe817e6b3f"},
+    {file = "ruff-0.1.1.tar.gz", hash = "sha256:c90461ae4abec261609e5ea436de4a4b5f2822921cf04c16d2cc9327182dbbcc"},
 ]
 
 [[package]]
@@ -5024,13 +5033,13 @@ zstd = ["zstandard (>=0.18.0)"]
 
 [[package]]
 name = "virtualenv"
-version = "20.24.5"
+version = "20.24.6"
 description = "Virtual Python Environment builder"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "virtualenv-20.24.5-py3-none-any.whl", hash = "sha256:b80039f280f4919c77b30f1c23294ae357c4c8701042086e3fc005963e4e537b"},
+    {file = "virtualenv-20.24.6-py3-none-any.whl", hash = "sha256:520d056652454c5098a00c0f073611ccbea4c79089331f60bf9d7ba247bb7381"},
-    {file = "virtualenv-20.24.5.tar.gz", hash = "sha256:e8361967f6da6fbdf1426483bfe9fca8287c242ac0bc30429905721cefbff752"},
+    {file = "virtualenv-20.24.6.tar.gz", hash = "sha256:02ece4f56fbf939dbbc33c0715159951d6bf14aaf5457b092e4548e1382455af"},
 ]
 
 [package.dependencies]
@@ -5179,4 +5188,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
 [metadata]
 lock-version = "2.0"
 python-versions = "~3.11"
-content-hash = "acb0688e485872194c21e1313e20fc4a67084893b26e9b8cde1d66e3fdbb1282"
+content-hash = "9538e574ca03437994b7b0a0b6cb41842256162a2f14abfd0da26587709f145a"

pyproject.toml

@@ -140,6 +140,7 @@ parameterized = "^0.8"
 pprofile = "*"
 pre-commit = "*"
 pygame = "*"
+pympler = "*"
 pyprof2calltree = "*"
 pytest = "*"
 pytest-cov = "*"

selfdrive/athena/athenad.py

@@ -20,7 +20,7 @@ from dataclasses import asdict, dataclass, replace
 from datetime import datetime
 from functools import partial
 from queue import Queue
-from typing import BinaryIO, Callable, Dict, List, Optional, Set, Union, cast
+from typing import Callable, Dict, List, Optional, Set, Union, cast
 
 import requests
 from jsonrpc import JSONRPCResponseManager, dispatcher
@@ -290,19 +290,15 @@ def _do_upload(upload_item: UploadItem, callback: Optional[Callable] = None) ->
     compress = True
 
   with open(path, "rb") as f:
-    data: BinaryIO
+    content = f.read()
     if compress:
       cloudlog.event("athena.upload_handler.compress", fn=path, fn_orig=upload_item.path)
-      compressed = bz2.compress(f.read())
+      content = bz2.compress(content)
-      size = len(compressed)
-      data = io.BytesIO(compressed)
-    else:
-      size = os.fstat(f.fileno()).st_size
-      data = f
 
+  with io.BytesIO(content) as data:
     return requests.put(upload_item.url,
-                        data=CallbackReader(data, callback, size) if callback else data,
+                        data=CallbackReader(data, callback, len(content)) if callback else data,
-                        headers={**upload_item.headers, 'Content-Length': str(size)},
+                        headers={**upload_item.headers, 'Content-Length': str(len(content))},
                         timeout=30)
 
 

selfdrive/athena/tests/test_athenad.py

@@ -9,10 +9,11 @@ import queue
 import unittest
 from dataclasses import asdict, replace
 from datetime import datetime, timedelta
+from parameterized import parameterized
 from typing import Optional
 
 from multiprocessing import Process
-from pathlib import Path
+from pympler.tracker import SummaryTracker
 from unittest import mock
 from websocket import ABNF
 from websocket._exceptions import WebSocketConnectionClosedException
@@ -57,10 +58,11 @@ class TestAthenadMethods(unittest.TestCase):
         break
 
   @staticmethod
-  def _create_file(file: str, parent: Optional[str] = None) -> str:
+  def _create_file(file: str, parent: Optional[str] = None, data: bytes = b'') -> str:
     fn = os.path.join(Paths.log_root() if parent is None else parent, file)
     os.makedirs(os.path.dirname(fn), exist_ok=True)
-    Path(fn).touch()
+    with open(fn, 'wb') as f:
+      f.write(data)
     return fn
 
 
@@ -137,19 +139,31 @@ class TestAthenadMethods(unittest.TestCase):
     if fn.endswith('.bz2'):
       self.assertEqual(athenad.strip_bz2_extension(fn), fn[:-4])
 
-
+  @parameterized.expand([(True,), (False,)])
   @with_http_server
-  def test_do_upload(self, host):
+  def test_do_upload(self, compress, host):
-    fn = self._create_file('qlog.bz2')
+    # random bytes to ensure rather large object post-compression
+    fn = self._create_file('qlog', data=os.urandom(10000 * 1024))
 
-    item = athenad.UploadItem(path=fn, url="http://localhost:1238", headers={}, created_at=int(time.time()*1000), id='')
+    # warm up object tracker
+    tracker = SummaryTracker()
+    for _ in range(5):
+      tracker.diff()
+
+    upload_fn = fn + ('.bz2' if compress else '')
+    item = athenad.UploadItem(path=upload_fn, url="http://localhost:1238", headers={}, created_at=int(time.time()*1000), id='')
     with self.assertRaises(requests.exceptions.ConnectionError):
       athenad._do_upload(item)
 
-    item = athenad.UploadItem(path=fn, url=f"{host}/qlog.bz2", headers={}, created_at=int(time.time()*1000), id='')
+    item = athenad.UploadItem(path=upload_fn, url=f"{host}/qlog.bz2", headers={}, created_at=int(time.time()*1000), id='')
     resp = athenad._do_upload(item)
     self.assertEqual(resp.status_code, 201)
 
+    # assert memory cleaned up
+    for _type, num_objects, total_size in tracker.diff():
+      with self.subTest(_type=_type):
+        self.assertLess(total_size / 1024, 10, f'Object {_type} ({num_objects=}) grew larger than 10 kB while uploading file')
+
   @with_http_server
   def test_uploadFileToUrl(self, host):
     fn = self._create_file('qlog.bz2')

selfdrive/car/honda/values.py

@@ -129,8 +129,8 @@ CAR_INFO: Dict[str, Optional[Union[HondaCarInfo, List[HondaCarInfo]]]] = {
   ],
   CAR.CIVIC_BOSCH_DIESEL: None,  # same platform
   CAR.CIVIC_2022: [
-    HondaCarInfo("Honda Civic 2022", "All", video_link="https://youtu.be/ytiOT5lcp6Q"),
+    HondaCarInfo("Honda Civic 2022-23", "All", video_link="https://youtu.be/ytiOT5lcp6Q"),
-    HondaCarInfo("Honda Civic Hatchback 2022", "All", video_link="https://youtu.be/ytiOT5lcp6Q"),
+    HondaCarInfo("Honda Civic Hatchback 2022-23", "All", video_link="https://youtu.be/ytiOT5lcp6Q"),
   ],
   CAR.ACURA_ILX: HondaCarInfo("Acura ILX 2016-19", "AcuraWatch Plus", min_steer_speed=25. * CV.MPH_TO_MS),
   CAR.CRV: HondaCarInfo("Honda CR-V 2015-16", "Touring Trim", min_steer_speed=12. * CV.MPH_TO_MS),
@@ -1519,6 +1519,7 @@ FW_VERSIONS = {
       b'77959-T47-A940\x00\x00',
       b'77959-T47-A950\x00\x00',
       b'77959-T20-M820\x00\x00',
+      b'77959-T20-A980\x00\x00',
     ],
     (Ecu.combinationMeter, 0x18DA60F1, None): [
       b'78108-T21-A220\x00\x00',
@@ -1527,6 +1528,7 @@ FW_VERSIONS = {
       b'78108-T21-A230\x00\x00',
       b'78108-T22-A020\x00\x00',
       b'78108-T21-MB10\x00\x00',
+      b'78108-T21-A740\x00\x00',
     ],
     (Ecu.fwdRadar, 0x18dab0f1, None): [
       b'36161-T20-A070\x00\x00',
@@ -1554,6 +1556,7 @@ FW_VERSIONS = {
       b'37805-64A-A540\x00\x00',
       b'37805-64A-A620\x00\x00',
       b'37805-64D-P510\x00\x00',
+      b'37805-64S-AA10\x00\x00',
     ],
   },
 }

selfdrive/modeld/modeld.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-import sys
 import os
 import time
 import pickle
@@ -17,12 +16,14 @@ from openpilot.common.numpy_fast import interp
 from openpilot.common.filter_simple import FirstOrderFilter
 from openpilot.common.realtime import config_realtime_process
 from openpilot.common.transformations.model import get_warp_matrix
+from openpilot.selfdrive import sentry
 from openpilot.selfdrive.modeld.runners import ModelRunner, Runtime
 from openpilot.selfdrive.modeld.parse_model_outputs import Parser
 from openpilot.selfdrive.modeld.fill_model_msg import fill_model_msg, fill_pose_msg, PublishState
 from openpilot.selfdrive.modeld.constants import ModelConstants
 from openpilot.selfdrive.modeld.models.commonmodel_pyx import ModelFrame, CLContext
 
+PROCESS_NAME = "selfdrive.modeld.modeld"
 SEND_RAW_PRED = os.getenv('SEND_RAW_PRED')
 
 MODEL_PATHS = {
@@ -113,8 +114,9 @@ class ModelState:
 
 
 def main():
-  cloudlog.bind(daemon="selfdrive.modeld.modeld")
+  sentry.set_tag("daemon", PROCESS_NAME)
-  setproctitle("selfdrive.modeld.modeld")
+  cloudlog.bind(daemon=PROCESS_NAME)
+  setproctitle(PROCESS_NAME)
   config_realtime_process(7, 54)
 
   cl_context = CLContext()
@@ -285,4 +287,7 @@ if __name__ == "__main__":
   try:
     main()
   except KeyboardInterrupt:
-    sys.exit()
+    cloudlog.warning(f"child {PROCESS_NAME} got SIGINT")
+  except Exception:
+    sentry.capture_exception()
+    raise

selfdrive/test/docker_common.sh

@@ -1,17 +1,17 @@
-if [ $1 = "base" ]; then
+if [ "$1" = "base" ]; then
   export DOCKER_IMAGE=openpilot-base
   export DOCKER_FILE=Dockerfile.openpilot_base
-elif [ $1 = "sim" ]; then
+elif [ "$1" = "sim" ]; then
   export DOCKER_IMAGE=openpilot-sim
   export DOCKER_FILE=tools/sim/Dockerfile.sim
-elif [ $1 = "prebuilt" ]; then
+elif [ "$1" = "prebuilt" ]; then
   export DOCKER_IMAGE=openpilot-prebuilt
   export DOCKER_FILE=Dockerfile.openpilot
-elif [ $1 = "cl" ]; then
+elif [ "$1" = "cl" ]; then
   export DOCKER_IMAGE=openpilot-base-cl
   export DOCKER_FILE=Dockerfile.openpilot_base_cl
 else
-  echo "Invalid docker build image $1"
+  echo "Invalid docker build image: '$1'"
   exit 1
 fi
 

selfdrive/test/process_replay/migration.py

@@ -94,6 +94,8 @@ def migrate_peripheralState(lr):
 def migrate_cameraStates(lr):
   all_msgs = []
   frame_to_encode_id = defaultdict(dict)
+  # just for encodeId fallback mechanism
+  min_frame_id = defaultdict(lambda: float('inf'))
 
   for msg in lr:
     if msg.which() not in ["roadEncodeIdx", "wideRoadEncodeIdx", "driverEncodeIdx"]:
@@ -111,10 +113,18 @@ def migrate_cameraStates(lr):
       continue
 
     camera_state = getattr(msg, msg.which())
+    min_frame_id[msg.which()] = min(min_frame_id[msg.which()], camera_state.frameId)
+
     encode_id = frame_to_encode_id[msg.which()].get(camera_state.frameId)
     if encode_id is None:
       print(f"Missing encoded frame for camera feed {msg.which()} with frameId: {camera_state.frameId}")
-      continue
+      if len(frame_to_encode_id[msg.which()]) != 0:
+        continue
+
+      # fallback mechanism for logs without encodeIdx (e.g. logs from before 2022 with dcamera recording disabled)
+      # try to fake encode_id by subtracting lowest frameId
+      encode_id = camera_state.frameId - min_frame_id[msg.which()]
+      print(f"Faking encodeId to {encode_id} for camera feed {msg.which()} with frameId: {camera_state.frameId}")
 
     new_msg = messaging.new_message(msg.which())
     new_camera_state = getattr(new_msg, new_msg.which())

selfdrive/test/process_replay/process_replay.py

@@ -27,6 +27,7 @@ from openpilot.selfdrive.test.process_replay.vision_meta import meta_from_camera
 from openpilot.selfdrive.test.process_replay.migration import migrate_all
 from openpilot.selfdrive.test.process_replay.capture import ProcessOutputCapture
 from openpilot.tools.lib.logreader import LogIterable
+from openpilot.tools.lib.framereader import BaseFrameReader
 
 # Numpy gives different results based on CPU features after version 19
 NUMPY_TOLERANCE = 1e-7
@@ -201,16 +202,15 @@ class ProcessContainer:
 
     self.environ_config = environ_config
 
-  def _setup_vision_ipc(self, all_msgs):
+  def _setup_vision_ipc(self, all_msgs: LogIterable, frs: Dict[str, Any]):
     assert len(self.cfg.vision_pubs) != 0
 
-    device_type = next(str(msg.initData.deviceType) for msg in all_msgs if msg.which() == "initData")
-
     vipc_server = VisionIpcServer("camerad")
     streams_metas = available_streams(all_msgs)
     for meta in streams_metas:
       if meta.camera_state in self.cfg.vision_pubs:
-        vipc_server.create_buffers(meta.stream, 2, False, *meta.frame_sizes[device_type])
+        frame_size = (frs[meta.camera_state].w, frs[meta.camera_state].h)
+        vipc_server.create_buffers(meta.stream, 2, False, *frame_size)
     vipc_server.start_listener()
 
     self.vipc_server = vipc_server
@@ -224,7 +224,7 @@ class ProcessContainer:
 
   def start(
     self, params_config: Dict[str, Any], environ_config: Dict[str, Any],
-    all_msgs: LogIterable,
+    all_msgs: LogIterable, frs: Optional[Dict[str, BaseFrameReader]],
     fingerprint: Optional[str], capture_output: bool
   ):
     with self.prefix as p:
@@ -241,7 +241,8 @@ class ProcessContainer:
       self.sockets = [messaging.sub_sock(s, timeout=100) for s in self.cfg.subs]
 
       if len(self.cfg.vision_pubs) != 0:
-        self._setup_vision_ipc(all_msgs)
+        assert frs is not None
+        self._setup_vision_ipc(all_msgs, frs)
         assert self.vipc_server is not None
 
       if capture_output:
@@ -265,7 +266,7 @@ class ProcessContainer:
       self.prefix.clean_dirs()
       self._clean_env()
 
-  def run_step(self, msg: capnp._DynamicStructReader, frs: Optional[Dict[str, Any]]) -> List[capnp._DynamicStructReader]:
+  def run_step(self, msg: capnp._DynamicStructReader, frs: Optional[Dict[str, BaseFrameReader]]) -> List[capnp._DynamicStructReader]:
     assert self.rc and self.pm and self.sockets and self.process.proc
 
     output_msgs = []
@@ -622,7 +623,7 @@ def replay_process_with_name(name: Union[str, Iterable[str]], lr: LogIterable, *
 
 
 def replay_process(
-  cfg: Union[ProcessConfig, Iterable[ProcessConfig]], lr: LogIterable, frs: Optional[Dict[str, Any]] = None,
+  cfg: Union[ProcessConfig, Iterable[ProcessConfig]], lr: LogIterable, frs: Optional[Dict[str, BaseFrameReader]] = None,
   fingerprint: Optional[str] = None, return_all_logs: bool = False, custom_params: Optional[Dict[str, Any]] = None,
   captured_output_store: Optional[Dict[str, Dict[str, str]]] = None, disable_progress: bool = False
 ) -> List[capnp._DynamicStructReader]:
@@ -650,7 +651,7 @@ def replay_process(
 
 
 def _replay_multi_process(
-  cfgs: List[ProcessConfig], lr: LogIterable, frs: Optional[Dict[str, Any]], fingerprint: Optional[str],
+  cfgs: List[ProcessConfig], lr: LogIterable, frs: Optional[Dict[str, BaseFrameReader]], fingerprint: Optional[str],
   custom_params: Optional[Dict[str, Any]], captured_output_store: Optional[Dict[str, Dict[str, str]]], disable_progress: bool
 ) -> List[capnp._DynamicStructReader]:
   if fingerprint is not None:
@@ -677,7 +678,7 @@ def _replay_multi_process(
     for cfg in cfgs:
       container = ProcessContainer(cfg)
       containers.append(container)
-      container.start(params_config, env_config, all_msgs, fingerprint, captured_output_store is not None)
+      container.start(params_config, env_config, all_msgs, frs, fingerprint, captured_output_store is not None)
 
     all_pubs = {pub for container in containers for pub in container.pubs}
     all_subs = {sub for container in containers for sub in container.subs}

selfdrive/test/process_replay/ref_commit

@@ -1 +1 @@
-d05b67ec66630521e8cedb90981002d57d738f6d
+6e27d5c97fe6554a86e9ee8bb9259e0cc6df5bb1

selfdrive/test/process_replay/regen.py

@@ -3,18 +3,42 @@ import os
 import argparse
 import time
 import capnp
+import numpy as np
 
 from typing import Union, Iterable, Optional, List, Any, Dict, Tuple
 
 from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, FAKEDATA, ProcessConfig, replay_process, get_process_config, \
                                                                    check_openpilot_enabled, get_custom_params_from_lr
+from openpilot.selfdrive.test.process_replay.vision_meta import DRIVER_FRAME_SIZES
 from openpilot.selfdrive.test.update_ci_routes import upload_route
 from openpilot.tools.lib.route import Route
-from openpilot.tools.lib.framereader import FrameReader
+from openpilot.tools.lib.framereader import FrameReader, BaseFrameReader, FrameType
 from openpilot.tools.lib.logreader import LogReader, LogIterable
 from openpilot.tools.lib.helpers import save_log
 
 
+class DummyFrameReader(BaseFrameReader):
+  def __init__(self, w: int, h: int, frame_count: int, pix_val: int):
+    self.pix_val = pix_val
+    self.w, self.h = w, h
+    self.frame_count = frame_count
+    self.frame_type = FrameType.raw
+
+  def get(self, idx, count=1, pix_fmt="yuv420p"):
+    if pix_fmt == "rgb24":
+      shape = (self.h, self.w, 3)
+    elif pix_fmt == "nv12" or pix_fmt == "yuv420p":
+      shape = (int((self.h * self.w) * 3 / 2),)
+    else:
+      raise NotImplementedError
+
+    return [np.full(shape, self.pix_val, dtype=np.uint8) for _ in range(count)]
+
+  @staticmethod
+  def zero_dcamera():
+    return DummyFrameReader(*DRIVER_FRAME_SIZES["tici"], 1200, 0)
+
+
 def regen_segment(
   lr: LogIterable, frs: Optional[Dict[str, Any]] = None,
   processes: Iterable[ProcessConfig] = CONFIGS, disable_tqdm: bool = False
@@ -31,7 +55,8 @@ def regen_segment(
 
 
 def setup_data_readers(
-    route: str, sidx: int, use_route_meta: bool, needs_driver_cam: bool = True, needs_road_cam: bool = True
+    route: str, sidx: int, use_route_meta: bool,
+    needs_driver_cam: bool = True, needs_road_cam: bool = True, dummy_driver_cam: bool = False
 ) -> Tuple[LogReader, Dict[str, Any]]:
   if use_route_meta:
     r = Route(route)
@@ -41,8 +66,13 @@ def setup_data_readers(
       frs['roadCameraState'] = FrameReader(r.camera_paths()[sidx])
     if needs_road_cam and  len(r.ecamera_paths()) > sidx and r.ecamera_paths()[sidx] is not None:
       frs['wideRoadCameraState'] = FrameReader(r.ecamera_paths()[sidx])
-    if needs_driver_cam and len(r.dcamera_paths()) > sidx and r.dcamera_paths()[sidx] is not None:
+    if needs_driver_cam:
-      frs['driverCameraState'] = FrameReader(r.dcamera_paths()[sidx])
+      if dummy_driver_cam:
+        frs['driverCameraState'] = DummyFrameReader.zero_dcamera()
+      elif len(r.dcamera_paths()) > sidx and r.dcamera_paths()[sidx] is not None:
+        device_type = next(str(msg.initData.deviceType) for msg in lr if msg.which() == "initData")
+        assert device_type != "neo", "Driver camera not supported on neo segments. Use dummy dcamera."
+        frs['driverCameraState'] = FrameReader(r.dcamera_paths()[sidx])
   else:
     lr = LogReader(f"cd:/{route.replace('|', '/')}/{sidx}/rlog.bz2")
     frs = {}
@@ -51,14 +81,19 @@ def setup_data_readers(
       if next((True for m in lr if m.which() == "wideRoadCameraState"), False):
         frs['wideRoadCameraState'] = FrameReader(f"cd:/{route.replace('|', '/')}/{sidx}/ecamera.hevc")
     if needs_driver_cam:
-      frs['driverCameraState'] = FrameReader(f"cd:/{route.replace('|', '/')}/{sidx}/dcamera.hevc")
+      if dummy_driver_cam:
+        frs['driverCameraState'] = DummyFrameReader.zero_dcamera()
+      else:
+        device_type = next(str(msg.initData.deviceType) for msg in lr if msg.which() == "initData")
+        assert device_type != "neo", "Driver camera not supported on neo segments. Use dummy dcamera."
+        frs['driverCameraState'] = FrameReader(f"cd:/{route.replace('|', '/')}/{sidx}/dcamera.hevc")
 
   return lr, frs
 
 
 def regen_and_save(
   route: str, sidx: int, processes: Union[str, Iterable[str]] = "all", outdir: str = FAKEDATA,
-  upload: bool = False, use_route_meta: bool = False, disable_tqdm: bool = False
+  upload: bool = False, use_route_meta: bool = False, disable_tqdm: bool = False, dummy_driver_cam: bool = False
 ) -> str:
   if not isinstance(processes, str) and not hasattr(processes, "__iter__"):
     raise ValueError("whitelist_proc must be a string or iterable")
@@ -77,7 +112,8 @@ def regen_and_save(
   all_vision_pubs = {pub for cfg in replayed_processes for pub in cfg.vision_pubs}
   lr, frs = setup_data_readers(route, sidx, use_route_meta,
                                needs_driver_cam="driverCameraState" in all_vision_pubs,
-                               needs_road_cam="roadCameraState" in all_vision_pubs or "wideRoadCameraState" in all_vision_pubs)
+                               needs_road_cam="roadCameraState" in all_vision_pubs or "wideRoadCameraState" in all_vision_pubs,
+                               dummy_driver_cam=dummy_driver_cam)
   output_logs = regen_segment(lr, frs, replayed_processes, disable_tqdm=disable_tqdm)
 
   log_dir = os.path.join(outdir, time.strftime("%Y-%m-%d--%H-%M-%S--0", time.gmtime()))
@@ -107,6 +143,7 @@ if __name__ == "__main__":
   parser = argparse.ArgumentParser(description="Generate new segments from old ones")
   parser.add_argument("--upload", action="store_true", help="Upload the new segment to the CI bucket")
   parser.add_argument("--outdir", help="log output dir", default=FAKEDATA)
+  parser.add_argument("--dummy-dcamera", action='store_true', help="Use dummy blank driver camera")
   parser.add_argument("--whitelist-procs", type=comma_separated_list, default=all_procs,
                       help="Comma-separated whitelist of processes to regen (e.g. controlsd,radard)")
   parser.add_argument("--blacklist-procs", type=comma_separated_list, default=[],
@@ -117,4 +154,4 @@ if __name__ == "__main__":
 
   blacklist_set = set(args.blacklist_procs)
   processes = [p for p in args.whitelist_procs if p not in blacklist_set]
-  regen_and_save(args.route, args.seg, processes=processes, upload=args.upload, outdir=args.outdir)
+  regen_and_save(args.route, args.seg, processes=processes, upload=args.upload, outdir=args.outdir, dummy_driver_cam=args.dummy_dcamera)

selfdrive/test/process_replay/regen_all.py

@@ -18,7 +18,7 @@ def regen_job(segment, upload, disable_tqdm):
     fake_dongle_id = 'regen' + ''.join(random.choice('0123456789ABCDEF') for _ in range(11))
     try:
       relr = regen_and_save(sn.route_name.canonical_name, sn.segment_num, upload=upload, use_route_meta=False,
-                            outdir=os.path.join(FAKEDATA, fake_dongle_id), disable_tqdm=disable_tqdm)
+                            outdir=os.path.join(FAKEDATA, fake_dongle_id), disable_tqdm=disable_tqdm, dummy_driver_cam=True)
       relr = '|'.join(relr.split('/')[-2:])
       return f'  ("{segment[0]}", "{relr}"), '
     except Exception as e:

selfdrive/test/process_replay/test_processes.py

@@ -41,23 +41,23 @@ source_segments = [
 ]
 
 segments = [
-  ("BODY", "aregenECF15D9E559|2023-05-10--14-26-40--0"),
+  ("BODY", "regen7FE9F3C7CE3|2023-10-25--23-56-32--0"),
-  ("HYUNDAI", "aregenAB9F543F70A|2023-05-10--14-28-25--0"),
+  ("HYUNDAI", "regen7519EF9EE71|2023-10-25--23-53-59--0"),
-  ("HYUNDAI2", "aregen39F5A028F96|2023-05-10--14-31-00--0"),
+  ("HYUNDAI2", "regenF68B9F1B286|2023-10-25--23-56-31--0"),
-  ("TOYOTA", "aregen8D6A8B36E8D|2023-05-10--14-32-38--0"),
+  ("TOYOTA", "regen56DC072FA51|2023-10-25--23-53-51--0"),
-  ("TOYOTA2", "aregenB1933C49809|2023-05-10--14-34-14--0"),
+  ("TOYOTA2", "regen78130056536|2023-10-25--23-53-58--0"),
-  ("TOYOTA3", "aregen5D9915223DC|2023-05-10--14-36-43--0"),
+  ("TOYOTA3", "regenC554B250909|2023-10-25--23-58-53--0"),
-  ("HONDA", "aregen484B732B675|2023-05-10--14-38-23--0"),
+  ("HONDA", "regen3ED625586FB|2023-10-25--23-56-29--0"),
-  ("HONDA2", "aregenAF6ACED4713|2023-05-10--14-40-01--0"),
+  ("HONDA2", "regen9F1A8F44FD5|2023-10-25--23-56-34--0"),
-  ("CHRYSLER", "aregen99B094E1E2E|2023-05-10--14-41-40--0"),
+  ("CHRYSLER", "regen60CE93181EA|2023-10-25--23-59-01--0"),
-  ("RAM", "aregen5C2487E1EEB|2023-05-10--14-44-09--0"),
+  ("RAM", "regen9E2B62E8E9A|2023-10-26--00-00-41--0"),
-  ("SUBARU", "aregen98D277B792E|2023-05-10--14-46-46--0"),
+  ("SUBARU", "regenEEBF379E0ED|2023-10-26--00-01-37--0"),
-  ("GM", "aregen377BA28D848|2023-05-10--14-48-28--0"),
+  ("GM", "regen0B0EE5D6E0D|2023-10-25--23-58-57--0"),
-  ("GM2", "aregen7CA0CC0F0C2|2023-05-10--14-51-00--0"),
+  ("GM2", "regen043B44E4FBD|2023-10-26--00-03-51--0"),
-  ("NISSAN", "aregen7097BF01563|2023-05-10--14-52-43--0"),
+  ("NISSAN", "regen14F35E327BC|2023-10-26--00-01-22--0"),
-  ("VOLKSWAGEN", "aregen765AF3D2CB5|2023-05-10--14-54-23--0"),
+  ("VOLKSWAGEN", "regen63A052AE7D7|2023-10-26--00-01-36--0"),
-  ("MAZDA", "aregen3053762FF2E|2023-05-10--14-56-53--0"),
+  ("MAZDA", "regenF9047685121|2023-10-26--00-05-02--0"),
-  ("FORD", "aregenDDE0F89FA1E|2023-05-10--14-59-26--0"),
+  ("FORD", "regen5115F2AE4FE|2023-10-26--00-06-17--0"),
   ]
 
 # dashcamOnly makes don't need to be tested until a full port is done

selfdrive/test/process_replay/test_regen.py

@@ -4,13 +4,12 @@ import unittest
 
 from parameterized import parameterized
 
-from openpilot.selfdrive.test.process_replay.regen import regen_segment
+from openpilot.selfdrive.test.process_replay.regen import regen_segment, DummyFrameReader
-from openpilot.selfdrive.test.process_replay.process_replay import check_openpilot_enabled, CONFIGS
+from openpilot.selfdrive.test.process_replay.process_replay import check_openpilot_enabled
 from openpilot.selfdrive.test.openpilotci import get_url
 from openpilot.tools.lib.logreader import LogReader
 from openpilot.tools.lib.framereader import FrameReader
 
-EXCLUDED_PROCESSES = {"dmonitoringd", "dmonitoringmodeld"}
 TESTED_SEGMENTS = [
   ("PRIUS_C2", "0982d79ebb0de295|2021-01-04--17-13-21--13"), # TOYOTA PRIUS 2017:     NEO, pandaStateDEPRECATED, no peripheralState, sensorEventsDEPRECATED
   # Enable these once regen on CI becomes faster or use them for different tests running controlsd in isolation
@@ -21,9 +20,9 @@ TESTED_SEGMENTS = [
 
 def ci_setup_data_readers(route, sidx):
   lr = LogReader(get_url(route, sidx, "rlog"))
-  # dm disabled
   frs = {
     'roadCameraState': FrameReader(get_url(route, sidx, "fcamera")),
+    'driverCameraState': DummyFrameReader.zero_dcamera()
   }
   if next((True for m in lr if m.which() == "wideRoadCameraState"), False):
     frs["wideRoadCameraState"] = FrameReader(get_url(route, sidx, "ecamera"))
@@ -34,11 +33,9 @@ def ci_setup_data_readers(route, sidx):
 class TestRegen(unittest.TestCase):
   @parameterized.expand(TESTED_SEGMENTS)
   def test_engaged(self, case_name, segment):
-    tested_procs = [p for p in CONFIGS if p.proc_name not in EXCLUDED_PROCESSES]
-
     route, sidx = segment.rsplit("--", 1)
     lr, frs = ci_setup_data_readers(route, sidx)
-    output_logs = regen_segment(lr, frs, processes=tested_procs, disable_tqdm=True)
+    output_logs = regen_segment(lr, frs, disable_tqdm=True)
 
     engaged = check_openpilot_enabled(output_logs)
     self.assertTrue(engaged, f"openpilot not engaged in {case_name}")

selfdrive/ui/qt/network/networking.cc

@@ -105,6 +105,7 @@ void Networking::showEvent(QShowEvent *event) {
 }
 
 void Networking::hideEvent(QHideEvent *event) {
+  main_layout->setCurrentWidget(wifiScreen);
   wifi->stop();
 }
 

system/sensord/rawgps/rawgpsd.py

@@ -190,7 +190,13 @@ def setup_quectel(diag: ModemDiag) -> bool:
 
   if gps_enabled():
     at_cmd("AT+QGPSEND")
-  #at_cmd("AT+QGPSDEL=0")
+
+  if "GPS_COLD_START" in os.environ:
+    # deletes all assistance
+    at_cmd("AT+QGPSDEL=0")
+  else:
+    # allow module to perform hot start
+    at_cmd("AT+QGPSDEL=1")
 
   # disable DPO power savings for more accuracy
   at_cmd("AT+QGPSCFG=\"dpoenable\",0")

tools/cabana/.gitignore

@@ -2,6 +2,5 @@ moc_*
 *.moc
 
 cabana
-settings
 dbc/car_fingerprint_to_dbc.json
 tests/test_cabana

tools/cabana/binaryview.cc

@@ -2,6 +2,7 @@
 
 #include <algorithm>
 
+#include <QDebug>
 #include <QFontDatabase>
 #include <QHeaderView>
 #include <QMouseEvent>
@@ -273,7 +274,7 @@ void BinaryViewModel::refresh() {
     row_count = can->lastMessage(msg_id).dat.size();
     items.resize(row_count * column_count);
   }
-  int valid_rows = std::min(can->lastMessage(msg_id).dat.size(), row_count);
+  int valid_rows = std::min<int>(can->lastMessage(msg_id).dat.size(), row_count);
   for (int i = 0; i < valid_rows * column_count; ++i) {
     items[i].valid = true;
   }
@@ -311,7 +312,7 @@ void BinaryViewModel::updateState() {
       int val = ((binary[i] >> (7 - j)) & 1) != 0 ? 1 : 0;
       // Bit update frequency based highlighting
       double offset = !item.sigs.empty() ? 50 : 0;
-      auto n = last_msg.bit_change_counts[i][7 - j];
+      auto n = last_msg.last_changes[i].bit_change_counts[j];
       double min_f = n == 0 ? offset : offset + 25;
       double alpha = std::clamp(offset + log2(1.0 + factor * (double)n / (double)last_msg.count) * scaler, min_f, max_f);
       auto color = item.bg_color;
@@ -334,13 +335,8 @@ QVariant BinaryViewModel::headerData(int section, Qt::Orientation orientation, i
 }
 
 QVariant BinaryViewModel::data(const QModelIndex &index, int role) const {
-  if (role == Qt::ToolTipRole) {
+  auto item = (const BinaryViewModel::Item *)index.internalPointer();
-    auto item = (const BinaryViewModel::Item *)index.internalPointer();
+  return role == Qt::ToolTipRole && item && !item->sigs.empty() ? signalToolTip(item->sigs.back()) : QVariant();
-    if (item && !item->sigs.empty()) {
-      return signalToolTip(item->sigs.back());
-    }
-  }
-  return {};
 }
 
 // BinaryItemDelegate
@@ -388,7 +384,7 @@ void BinaryItemDelegate::paint(QPainter *painter, const QStyleOptionViewItem &op
           drawSignalCell(painter, option, index, s);
         }
       }
-    } else if (item->valid) {
+    } else if (item->valid && item->bg_color.alpha() > 0) {
       painter->fillRect(option.rect, item->bg_color);
     }
     auto color_role = item->sigs.contains(bin_view->hovered_sig) ? QPalette::BrightText : QPalette::Text;

tools/cabana/cabana.cc

@@ -18,8 +18,6 @@ int main(int argc, char *argv[]) {
   app.setWindowIcon(QIcon(":cabana-icon.png"));
 
   UnixSignalHandler signalHandler;
-
-  settings.load();
   utils::setTheme(settings.theme);
 
   QCommandLineParser cmd_parser;
@@ -28,6 +26,7 @@ int main(int argc, char *argv[]) {
   cmd_parser.addOption({"demo", "use a demo route instead of providing your own"});
   cmd_parser.addOption({"qcam", "load qcamera"});
   cmd_parser.addOption({"ecam", "load wide road camera"});
+  cmd_parser.addOption({"dcam", "load driver camera"});
   cmd_parser.addOption({"stream", "read can messages from live streaming"});
   cmd_parser.addOption({"panda", "read can messages from panda"});
   cmd_parser.addOption({"panda-serial", "read can messages from panda with given serial", "panda-serial"});
@@ -62,13 +61,10 @@ int main(int argc, char *argv[]) {
     stream = new SocketCanStream(&app, config);
   } else {
     uint32_t replay_flags = REPLAY_FLAG_NONE;
-    if (cmd_parser.isSet("ecam")) {
+    if (cmd_parser.isSet("ecam")) replay_flags |= REPLAY_FLAG_ECAM;
-      replay_flags |= REPLAY_FLAG_ECAM;
+    if (cmd_parser.isSet("qcam")) replay_flags |= REPLAY_FLAG_QCAMERA;
-    } else if (cmd_parser.isSet("qcam")) {
+    if (cmd_parser.isSet("dcam")) replay_flags |= REPLAY_FLAG_DCAM;
-      replay_flags |= REPLAY_FLAG_QCAMERA;
+    if (cmd_parser.isSet("no-vipc")) replay_flags |= REPLAY_FLAG_NO_VIPC;
-    } else if (cmd_parser.isSet("no-vipc")) {
-      replay_flags |= REPLAY_FLAG_NO_VIPC;
-    }
 
     const QStringList args = cmd_parser.positionalArguments();
     QString route;

tools/cabana/chart/chart.cc

@@ -279,38 +279,50 @@ void ChartView::updateSeriesPoints() {
   }
 }
 
-void ChartView::updateSeries(const cabana::Signal *sig, bool clear) {
+void ChartView::appendCanEvents(const cabana::Signal *sig, const std::vector<const CanEvent *> &events,
+                                std::vector<QPointF> &vals, std::vector<QPointF> &step_vals) {
+  vals.reserve(vals.size() + events.capacity());
+  step_vals.reserve(step_vals.size() + events.capacity() * 2);
+
+  double value = 0;
+  const uint64_t begin_mono_time = can->routeStartTime() * 1e9;
+  for (const CanEvent *e : events) {
+    if (sig->getValue(e->dat, e->size, &value)) {
+      const double ts = (e->mono_time - std::min(e->mono_time, begin_mono_time)) / 1e9;
+      vals.emplace_back(ts, value);
+      if (!step_vals.empty())
+        step_vals.emplace_back(ts, step_vals.back().y());
+      step_vals.emplace_back(ts, value);
+    }
+  }
+}
+
+void ChartView::updateSeries(const cabana::Signal *sig, const MessageEventsMap *msg_new_events) {
   for (auto &s : sigs) {
     if (!sig || s.sig == sig) {
-      if (clear) {
+      if (!msg_new_events) {
         s.vals.clear();
         s.step_vals.clear();
-        s.last_value_mono_time = 0;
       }
+      auto events = msg_new_events ? msg_new_events : &can->eventsMap();
+      auto it = events->find(s.msg_id);
+      if (it == events->end() || it->second.empty()) continue;
 
-      const auto &msgs = can->events(s.msg_id);
+      if (s.vals.empty() || (it->second.back()->mono_time / 1e9 - can->routeStartTime()) > s.vals.back().x()) {
-      s.vals.reserve(msgs.capacity());
+        appendCanEvents(s.sig, it->second, s.vals, s.step_vals);
-      s.step_vals.reserve(msgs.capacity() * 2);
+      } else {
-
+        std::vector<QPointF> vals, step_vals;
-      auto first = std::upper_bound(msgs.cbegin(), msgs.cend(), s.last_value_mono_time, CompareCanEvent());
+        appendCanEvents(s.sig, it->second, vals, step_vals);
-      const double route_start_time = can->routeStartTime();
+        s.vals.insert(std::lower_bound(s.vals.begin(), s.vals.end(), vals.front().x(), xLessThan),
-      for (auto end = msgs.cend(); first != end; ++first) {
+                      vals.begin(), vals.end());
-        const CanEvent *e = *first;
+        s.step_vals.insert(std::lower_bound(s.step_vals.begin(), s.step_vals.end(), step_vals.front().x(), xLessThan),
-        double value = 0;
+                           step_vals.begin(), step_vals.end());
-        if (s.sig->getValue(e->dat, e->size, &value)) {
-          double ts = e->mono_time / 1e9 - route_start_time;  // seconds
-          s.vals.append({ts, value});
-          if (!s.step_vals.empty()) {
-            s.step_vals.append({ts, s.step_vals.back().y()});
-          }
-          s.step_vals.append({ts, value});
-          s.last_value_mono_time = e->mono_time;
-        }
       }
+
       if (!can->liveStreaming()) {
         s.segment_tree.build(s.vals);
       }
-      s.series->replace(series_type == SeriesType::StepLine ? s.step_vals : s.vals);
+      s.series->replace(QVector<QPointF>::fromStdVector(series_type == SeriesType::StepLine ? s.step_vals : s.vals));
     }
   }
   updateAxisY();
@@ -320,7 +332,7 @@ void ChartView::updateSeries(const cabana::Signal *sig, bool clear) {
 
 // auto zoom on yaxis
 void ChartView::updateAxisY() {
-  if (sigs.isEmpty()) return;
+  if (sigs.empty()) return;
 
   double min = std::numeric_limits<double>::max();
   double max = std::numeric_limits<double>::lowest();
@@ -344,9 +356,7 @@ void ChartView::updateAxisY() {
         if (it->y() > s.max) s.max = it->y();
       }
     } else {
-      auto [min_y, max_y] = s.segment_tree.minmax(std::distance(s.vals.cbegin(), first), std::distance(s.vals.cbegin(), last));
+      std::tie(s.min, s.max) = s.segment_tree.minmax(std::distance(s.vals.cbegin(), first), std::distance(s.vals.cbegin(), last));
-      s.min = min_y;
-      s.max = max_y;
     }
     min = std::min(min, s.min);
     max = std::max(max, s.max);
@@ -365,7 +375,7 @@ void ChartView::updateAxisY() {
     axis_y->setRange(min_y, max_y);
     axis_y->setTickCount(tick_count);
 
-    int n = std::max(int(-std::floor(std::log10((max_y - min_y) / (tick_count - 1)))), 0) + 1;
+    int n = std::max(int(-std::floor(std::log10((max_y - min_y) / (tick_count - 1)))), 0);
     int max_label_width = 0;
     QFontMetrics fm(axis_y->labelsFont());
     for (int i = 0; i < tick_count; i++) {
@@ -453,7 +463,7 @@ static QPixmap getDropPixmap(const QPixmap &src) {
   return px;
 }
 
-void ChartView::contextMenuEvent(QContextMenuEvent *event) { 
+void ChartView::contextMenuEvent(QContextMenuEvent *event) {
   QMenu context_menu(this);
   context_menu.addActions(menu->actions());
   context_menu.addSeparator();
@@ -492,13 +502,9 @@ void ChartView::mouseReleaseEvent(QMouseEvent *event) {
   if (event->button() == Qt::LeftButton && rubber && rubber->isVisible()) {
     rubber->hide();
     auto rect = rubber->geometry().normalized();
-    double min = chart()->mapToValue(rect.topLeft()).x();
-    double max = chart()->mapToValue(rect.bottomRight()).x();
-
     // Prevent zooming/seeking past the end of the route
-    min = std::clamp(min, 0., can->totalSeconds());
+    double min = std::clamp(chart()->mapToValue(rect.topLeft()).x(), 0., can->totalSeconds());
-    max = std::clamp(max, 0., can->totalSeconds());
+    double max = std::clamp(chart()->mapToValue(rect.bottomRight()).x(), 0., can->totalSeconds());
-
     if (rubber->width() <= 0) {
       // no rubber dragged, seek to mouse position
       can->seekTo(min);
@@ -623,7 +629,7 @@ void ChartView::dropEvent(QDropEvent *event) {
         source_chart->chart()->removeSeries(s.series);
         addSeries(s.series);
       }
-      sigs.append(source_chart->sigs);
+      sigs.insert(sigs.end(), std::move_iterator(source_chart->sigs.begin()), std::move_iterator(source_chart->sigs.end()));
       updateAxisY();
       updateTitle();
       startAnimation();
@@ -763,13 +769,8 @@ void ChartView::drawSignalValue(QPainter *painter) {
   painter->setPen(chart()->legend()->labelColor());
   int i = 0;
   for (auto &s : sigs) {
-    QString value = "--";
+    auto it = std::lower_bound(s.vals.crbegin(), s.vals.crend(), cur_sec, [](auto &p, double x) { return p.x() > x; });
-    if (s.series->isVisible()) {
+    QString value = (it != s.vals.crend() && it->x() >= axis_x->min()) ? s.sig->formatValue(it->y()) : "--";
-      auto it = std::lower_bound(s.vals.crbegin(), s.vals.crend(), cur_sec, [](auto &p, double x) { return p.x() > x; });
-      if (it != s.vals.crend() && it->x() >= axis_x->min()) {
-        value = s.sig->formatValue(it->y());
-      }
-    }
     QRectF marker_rect = legend_markers[i++]->sceneBoundingRect();
     QRectF value_rect(marker_rect.bottomLeft() - QPoint(0, 1), marker_rect.size());
     QString elided_val = painter->fontMetrics().elidedText(value, Qt::ElideRight, value_rect.width());
@@ -841,9 +842,8 @@ void ChartView::setSeriesType(SeriesType type) {
       s.series->deleteLater();
     }
     for (auto &s : sigs) {
-      auto series = createSeries(series_type, s.sig->color);
+      s.series = createSeries(series_type, s.sig->color);
-      series->replace(series_type == SeriesType::StepLine ? s.step_vals : s.vals);
+      s.series->replace(QVector<QPointF>::fromStdVector(series_type == SeriesType::StepLine ? s.step_vals : s.vals));
-      s.series = series;
     }
     updateSeriesPoints();
     updateTitle();

tools/cabana/chart/chart.h

@@ -2,6 +2,7 @@
 
 #include <tuple>
 #include <utility>
+#include <vector>
 
 #include <QMenu>
 #include <QGraphicsPixmapItem>
@@ -31,7 +32,7 @@ public:
   ChartView(const std::pair<double, double> &x_range, ChartsWidget *parent = nullptr);
   void addSignal(const MessageId &msg_id, const cabana::Signal *sig);
   bool hasSignal(const MessageId &msg_id, const cabana::Signal *sig) const;
-  void updateSeries(const cabana::Signal *sig = nullptr, bool clear = true);
+  void updateSeries(const cabana::Signal *sig = nullptr, const MessageEventsMap *msg_new_events = nullptr);
   void updatePlot(double cur, double min, double max);
   void setSeriesType(SeriesType type);
   void updatePlotArea(int left, bool force = false);
@@ -43,9 +44,8 @@ public:
     MessageId msg_id;
     const cabana::Signal *sig = nullptr;
     QXYSeries *series = nullptr;
-    QVector<QPointF> vals;
+    std::vector<QPointF> vals;
-    QVector<QPointF> step_vals;
+    std::vector<QPointF> step_vals;
-    uint64_t last_value_mono_time = 0;
     QPointF track_pt{};
     SegmentTree segment_tree;
     double min = 0;
@@ -64,6 +64,8 @@ private slots:
   void signalRemoved(const cabana::Signal *sig) { removeIf([=](auto &s) { return s.sig == sig; }); }
 
 private:
+  void appendCanEvents(const cabana::Signal *sig, const std::vector<const CanEvent *> &events,
+                       std::vector<QPointF> &vals, std::vector<QPointF> &step_vals);
   void createToolButtons();
   void addSeries(QXYSeries *series);
   void contextMenuEvent(QContextMenuEvent *event) override;
@@ -107,7 +109,7 @@ private:
   QGraphicsProxyWidget *close_btn_proxy;
   QGraphicsProxyWidget *manage_btn_proxy;
   TipLabel tip_label;
-  QList<SigItem> sigs;
+  std::vector<SigItem> sigs;
   double cur_sec = 0;
   SeriesType series_type = SeriesType::Line;
   bool is_scrubbing = false;

tools/cabana/chart/chartswidget.cc

@@ -12,7 +12,7 @@
 #include "tools/cabana/chart/chart.h"
 
 const int MAX_COLUMN_COUNT = 4;
-const int CHART_SPACING = 10;
+const int CHART_SPACING = 4;
 
 ChartsWidget::ChartsWidget(QWidget *parent) : align_timer(this), auto_scroll_timer(this), QFrame(parent) {
   setFrameStyle(QFrame::StyledPanel | QFrame::Plain);
@@ -78,8 +78,9 @@ ChartsWidget::ChartsWidget(QWidget *parent) : align_timer(this), auto_scroll_tim
 
   // charts
   charts_container = new ChartsContainer(this);
-
+  charts_container->setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Fixed);
   charts_scroll = new QScrollArea(this);
+  charts_scroll->viewport()->setBackgroundRole(QPalette::Base);
   charts_scroll->setFrameStyle(QFrame::NoFrame);
   charts_scroll->setWidgetResizable(true);
   charts_scroll->setWidget(charts_container);
@@ -99,7 +100,7 @@ ChartsWidget::ChartsWidget(QWidget *parent) : align_timer(this), auto_scroll_tim
   QObject::connect(&auto_scroll_timer, &QTimer::timeout, this, &ChartsWidget::doAutoScroll);
   QObject::connect(dbc(), &DBCManager::DBCFileChanged, this, &ChartsWidget::removeAll);
   QObject::connect(can, &AbstractStream::eventsMerged, this, &ChartsWidget::eventsMerged);
-  QObject::connect(can, &AbstractStream::updated, this, &ChartsWidget::updateState);
+  QObject::connect(can, &AbstractStream::msgsReceived, this, &ChartsWidget::updateState);
   QObject::connect(range_slider, &QSlider::valueChanged, this, &ChartsWidget::setMaxChartRange);
   QObject::connect(new_plot_btn, &QToolButton::clicked, this, &ChartsWidget::newChart);
   QObject::connect(remove_all_btn, &QToolButton::clicked, this, &ChartsWidget::removeAll);
@@ -149,11 +150,10 @@ void ChartsWidget::updateTabBar() {
   }
 }
 
-void ChartsWidget::eventsMerged() {
+void ChartsWidget::eventsMerged(const MessageEventsMap &new_events) {
   QFutureSynchronizer<void> future_synchronizer;
-  bool clear = !can->liveStreaming();
   for (auto c : charts) {
-    future_synchronizer.addFuture(QtConcurrent::run(c, &ChartView::updateSeries, nullptr, clear));
+    future_synchronizer.addFuture(QtConcurrent::run(c, &ChartView::updateSeries, nullptr, &new_events));
   }
 }
 
@@ -190,7 +190,7 @@ void ChartsWidget::updateState() {
     if (pos < 0 || pos > 0.8) {
       display_range.first = std::max(0.0, cur_sec - max_chart_range * 0.1);
     }
-    double max_sec = std::min(std::floor(display_range.first + max_chart_range), can->totalSeconds());
+    double max_sec = std::min(display_range.first + max_chart_range, can->totalSeconds());
     display_range.first = std::max(0.0, max_sec - max_chart_range);
     display_range.second = display_range.first + max_chart_range;
   } else if (cur_sec < (zoomed_range.first - 0.1) || cur_sec >= zoomed_range.second) {
@@ -282,7 +282,7 @@ void ChartsWidget::splitChart(ChartView *src_chart) {
       it->series->setColor(it->sig->color);
 
       c->addSeries(it->series);
-      c->sigs.push_back(*it);
+      c->sigs.emplace_back(std::move(*it));
       c->updateAxisY();
       c->updateTitle();
       it = src_chart->sigs.erase(it);
@@ -322,9 +322,9 @@ void ChartsWidget::updateLayout(bool force) {
     }
     for (int i = 0; i < current_charts.size(); ++i) {
       charts_layout->addWidget(current_charts[i], i / n, i % n);
-      if (current_charts[i]->sigs.isEmpty()) {
+      if (current_charts[i]->sigs.empty()) {
         // the chart will be resized after add signal. delay setVisible to reduce flicker.
-        QTimer::singleShot(0, [c = current_charts[i]]() { c->setVisible(true); });
+        QTimer::singleShot(0, current_charts[i], [c = current_charts[i]]() { c->setVisible(true); });
       } else {
         current_charts[i]->setVisible(true);
       }
@@ -474,8 +474,9 @@ bool ChartsWidget::event(QEvent *event) {
 
 ChartsContainer::ChartsContainer(ChartsWidget *parent) : charts_widget(parent), QWidget(parent) {
   setAcceptDrops(true);
+  setBackgroundRole(QPalette::Window);
   QVBoxLayout *charts_main_layout = new QVBoxLayout(this);
-  charts_main_layout->setContentsMargins(0, 10, 0, 0);
+  charts_main_layout->setContentsMargins(0, CHART_SPACING, 0, CHART_SPACING);
   charts_layout = new QGridLayout();
   charts_layout->setSpacing(CHART_SPACING);
   charts_main_layout->addLayout(charts_layout);
@@ -519,15 +520,11 @@ void ChartsContainer::paintEvent(QPaintEvent *ev) {
       r.setHeight(CHART_SPACING);
     }
 
-    const int margin = (CHART_SPACING - 2) / 2;
-    QPainterPath path;
-    path.addPolygon(QPolygonF({r.topLeft(), QPointF(r.left() + CHART_SPACING, r.top() + r.height() / 2), r.bottomLeft()}));
-    path.addPolygon(QPolygonF({r.topRight(), QPointF(r.right() - CHART_SPACING, r.top() + r.height() / 2), r.bottomRight()}));
-
     QPainter p(this);
-    p.setRenderHint(QPainter::Antialiasing);
+    p.setPen(QPen(palette().highlight(), 2));
-    p.fillPath(path, palette().highlight());
+    p.drawLine(r.topLeft() + QPoint(1, 0), r.bottomLeft() + QPoint(1, 0));
-    p.fillRect(r.adjusted(2, margin, -2, -margin), palette().highlight());
+    p.drawLine(r.topLeft() + QPoint(0, r.height() / 2), r.topRight() + QPoint(0, r.height() / 2));
+    p.drawLine(r.topRight(), r.bottomRight());
   }
 }
 

tools/cabana/chart/chartswidget.h

@@ -63,7 +63,7 @@ private:
   void removeChart(ChartView *chart);
   void splitChart(ChartView *chart);
   QRect chartVisibleRect(ChartView *chart);
-  void eventsMerged();
+  void eventsMerged(const MessageEventsMap &new_events);
   void updateState();
   void zoomReset();
   void startAutoScroll();

tools/cabana/chart/signalselector.cc

@@ -44,9 +44,9 @@ SignalSelector::SignalSelector(QString title, QWidget *parent) : QDialog(parent)
   auto buttonBox = new QDialogButtonBox(QDialogButtonBox::Ok | QDialogButtonBox::Cancel);
   main_layout->addWidget(buttonBox, 3, 2);
 
-  for (auto it = can->last_msgs.cbegin(); it != can->last_msgs.cend(); ++it) {
+  for (const auto &[id, _] : can->lastMessages()) {
-    if (auto m = dbc()->msg(it.key())) {
+    if (auto m = dbc()->msg(id)) {
-      msgs_combo->addItem(QString("%1 (%2)").arg(m->name).arg(it.key().toString()), QVariant::fromValue(it.key()));
+      msgs_combo->addItem(QString("%1 (%2)").arg(m->name).arg(id.toString()), QVariant::fromValue(id));
     }
   }
   msgs_combo->model()->sort(0);

tools/cabana/dbc/dbcfile.cc

@@ -4,10 +4,8 @@
 #include <QFileInfo>
 #include <QRegularExpression>
 #include <QTextStream>
-#include <numeric>
-#include <sstream>
 
-DBCFile::DBCFile(const QString &dbc_file_name, QObject *parent) : QObject(parent) {
+DBCFile::DBCFile(const QString &dbc_file_name) {
   QFile file(dbc_file_name);
   if (file.open(QIODevice::ReadOnly)) {
     name_ = QFileInfo(dbc_file_name).baseName();
@@ -22,7 +20,7 @@ DBCFile::DBCFile(const QString &dbc_file_name, QObject *parent) : QObject(parent
   }
 }
 
-DBCFile::DBCFile(const QString &name, const QString &content, QObject *parent) : QObject(parent), name_(name), filename("") {
+DBCFile::DBCFile(const QString &name, const QString &content) : name_(name), filename("") {
   // Open from clipboard
   parse(content);
 }

tools/cabana/dbc/dbcfile.h

@@ -1,18 +1,15 @@
 #pragma once
 
 #include <map>
-#include <QObject>
 
 #include "tools/cabana/dbc/dbc.h"
 
 const QString AUTO_SAVE_EXTENSION = ".tmp";
 
-class DBCFile : public QObject {
+class DBCFile {
-  Q_OBJECT
-
 public:
-  DBCFile(const QString &dbc_file_name, QObject *parent=nullptr);
+  DBCFile(const QString &dbc_file_name);
-  DBCFile(const QString &name, const QString &content, QObject *parent=nullptr);
+  DBCFile(const QString &name, const QString &content);
   ~DBCFile() {}
 
   bool save();

tools/cabana/dbc/dbcmanager.cc

@@ -7,7 +7,7 @@ bool DBCManager::open(const SourceSet &sources, const QString &dbc_file_name, QS
   try {
     auto it = std::find_if(dbc_files.begin(), dbc_files.end(),
                            [&](auto &f) { return f.second && f.second->filename == dbc_file_name; });
-    auto file = (it != dbc_files.end()) ? it->second : std::make_shared<DBCFile>(dbc_file_name, this);
+    auto file = (it != dbc_files.end()) ? it->second : std::make_shared<DBCFile>(dbc_file_name);
     for (auto s : sources) {
       dbc_files[s] = file;
     }
@@ -22,7 +22,7 @@ bool DBCManager::open(const SourceSet &sources, const QString &dbc_file_name, QS
 
 bool DBCManager::open(const SourceSet &sources, const QString &name, const QString &content, QString *error) {
   try {
-    auto file = std::make_shared<DBCFile>(name, content, this);
+    auto file = std::make_shared<DBCFile>(name, content);
     for (auto s : sources) {
       dbc_files[s] = file;
     }
@@ -189,6 +189,13 @@ const SourceSet DBCManager::sources(const DBCFile *dbc_file) const {
   return sources;
 }
 
+QString toString(const SourceSet &ss) {
+  return std::accumulate(ss.cbegin(), ss.cend(), QString(), [](QString str, int source) {
+    if (!str.isEmpty()) str += ", ";
+    return str + (source == -1 ? QStringLiteral("all") : QString::number(source));
+  });
+}
+
 DBCManager *dbc() {
   static DBCManager dbc_manager(nullptr);
   return &dbc_manager;

tools/cabana/dbc/dbcmanager.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <QObject>
 #include <memory>
 #include <map>
 #include <set>
@@ -66,15 +67,8 @@ private:
 
 DBCManager *dbc();
 
+QString toString(const SourceSet &ss);
 inline QString msgName(const MessageId &id) {
   auto msg = dbc()->msg(id);
   return msg ? msg->name : UNTITLED;
 }
-
-inline QString toString(const SourceSet &ss) {
-  QStringList ret;
-  for (auto s : ss) {
-    ret << (s == -1 ? QString("all") : QString::number(s));
-  }
-  return ret.join(", ");
-}

tools/cabana/detailwidget.cc

@@ -2,6 +2,7 @@
 
 #include <QFormLayout>
 #include <QMenu>
+#include <QSpacerItem>
 
 #include "tools/cabana/commands.h"
 #include "tools/cabana/mainwin.h"
@@ -22,19 +23,15 @@ DetailWidget::DetailWidget(ChartsWidget *charts, QWidget *parent) : charts(chart
   // message title
   QHBoxLayout *title_layout = new QHBoxLayout();
   title_layout->setContentsMargins(3, 6, 3, 0);
-  time_label = new QLabel(this);
+  auto spacer = new QSpacerItem(0, 1);
-  time_label->setToolTip(tr("Current time"));
+  title_layout->addItem(spacer);
-  time_label->setStyleSheet("QLabel{font-weight:bold;}");
+  title_layout->addWidget(name_label = new ElidedLabel(this), 1);
-  title_layout->addWidget(time_label);
-  name_label = new ElidedLabel(this);
   name_label->setStyleSheet("QLabel{font-weight:bold;}");
   name_label->setAlignment(Qt::AlignCenter);
-  name_label->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Preferred);
-  title_layout->addWidget(name_label);
   auto edit_btn = new ToolButton("pencil", tr("Edit Message"));
   title_layout->addWidget(edit_btn);
-  remove_btn = new ToolButton("x-lg", tr("Remove Message"));
+  title_layout->addWidget(remove_btn = new ToolButton("x-lg", tr("Remove Message")));
-  title_layout->addWidget(remove_btn);
+  spacer->changeSize(edit_btn->sizeHint().width() * 2 + 9, 1);
   main_layout->addLayout(title_layout);
 
   // warning
@@ -150,9 +147,8 @@ void DetailWidget::refresh() {
   warning_widget->setVisible(!warnings.isEmpty());
 }
 
-void DetailWidget::updateState(const QHash<MessageId, CanData> *msgs) {
+void DetailWidget::updateState(const std::set<MessageId> *msgs) {
-  time_label->setText(QString::number(can->currentSec(), 'f', 3));
+  if ((msgs && !msgs->count(msg_id)))
-  if ((msgs && !msgs->contains(msg_id)))
     return;
 
   if (tab_widget->currentIndex() == 0)

tools/cabana/detailwidget.h

@@ -4,6 +4,7 @@
 #include <QSplitter>
 #include <QTabWidget>
 #include <QTextEdit>
+#include <set>
 
 #include "selfdrive/ui/qt/widgets/controls.h"
 #include "tools/cabana/binaryview.h"
@@ -11,7 +12,6 @@
 #include "tools/cabana/historylog.h"
 #include "tools/cabana/signalview.h"
 
-class MainWindow;
 class EditMessageDialog : public QDialog {
 public:
   EditMessageDialog(const MessageId &msg_id, const QString &title, int size, QWidget *parent);
@@ -39,10 +39,10 @@ private:
   void showTabBarContextMenu(const QPoint &pt);
   void editMsg();
   void removeMsg();
-  void updateState(const QHash<MessageId, CanData> * msgs = nullptr);
+  void updateState(const std::set<MessageId> *msgs = nullptr);
 
   MessageId msg_id;
-  QLabel *time_label, *warning_icon, *warning_label;
+  QLabel *warning_icon, *warning_label;
   ElidedLabel *name_label;
   QWidget *warning_widget;
   TabBar *tabbar;

tools/cabana/historylog.cc

@@ -7,7 +7,6 @@
 #include <QVBoxLayout>
 
 #include "tools/cabana/commands.h"
-// HistoryLogModel
 
 QVariant HistoryLogModel::data(const QModelIndex &index, int role) const {
   const bool show_signals = display_signals_mode && sigs.size() > 0;
@@ -17,11 +16,11 @@ QVariant HistoryLogModel::data(const QModelIndex &index, int role) const {
       return QString::number((m.mono_time / (double)1e9) - can->routeStartTime(), 'f', 2);
     }
     int i = index.column() - 1;
-    return show_signals ? QString::number(m.sig_values[i], 'f', sigs[i]->precision) : toHex(m.data);
+    return show_signals ? QString::number(m.sig_values[i], 'f', sigs[i]->precision) : QString();
   } else if (role == ColorsRole) {
-    return QVariant::fromValue(m.colors);
+    return QVariant::fromValue((void *)(&m.colors));
   } else if (role == BytesRole) {
-    return m.data;
+    return QVariant::fromValue((void *)(&m.data));
   } else if (role == Qt::TextAlignmentRole) {
     return (uint32_t)(Qt::AlignRight | Qt::AlignVCenter);
   }
@@ -123,7 +122,7 @@ void HistoryLogModel::fetchMore(const QModelIndex &parent) {
 template <class InputIt>
 std::deque<HistoryLogModel::Message> HistoryLogModel::fetchData(InputIt first, InputIt last, uint64_t min_time) {
   std::deque<HistoryLogModel::Message> msgs;
-  QVector<double> values(sigs.size());
+  std::vector<double> values(sigs.size());
   for (; first != last && (*first)->mono_time > min_time; ++first) {
     const CanEvent *e = *first;
     for (int i = 0; i < sigs.size(); ++i) {
@@ -132,7 +131,7 @@ std::deque<HistoryLogModel::Message> HistoryLogModel::fetchData(InputIt first, I
     if (!filter_cmp || filter_cmp(values[filter_sig_idx], filter_value)) {
       auto &m = msgs.emplace_back();
       m.mono_time = e->mono_time;
-      m.data = QByteArray((const char *)e->dat, e->size);
+      m.data.assign(e->dat, e->dat + e->size);
       m.sig_values = values;
       if (msgs.size() >= batch_size && min_time == 0) {
         return msgs;
@@ -146,7 +145,7 @@ std::deque<HistoryLogModel::Message> HistoryLogModel::fetchData(uint64_t from_ti
   const auto &events = can->events(msg_id);
   const auto freq = can->lastMessage(msg_id).freq;
   const bool update_colors = !display_signals_mode || sigs.empty();
-
+  const std::vector<uint8_t> no_mask;
   const auto speed = can->getSpeed();
   if (dynamic_mode) {
     auto first = std::upper_bound(events.rbegin(), events.rend(), from_time, [](uint64_t ts, auto e) {
@@ -155,7 +154,7 @@ std::deque<HistoryLogModel::Message> HistoryLogModel::fetchData(uint64_t from_ti
     auto msgs = fetchData(first, events.rend(), min_time);
     if (update_colors && (min_time > 0 || messages.empty())) {
       for (auto it = msgs.rbegin(); it != msgs.rend(); ++it) {
-        hex_colors.compute(msg_id, it->data.data(), it->data.size(), it->mono_time / (double)1e9, speed, nullptr, freq);
+        hex_colors.compute(msg_id, it->data.data(), it->data.size(), it->mono_time / (double)1e9, speed, no_mask, freq);
         it->colors = hex_colors.colors;
       }
     }
@@ -166,7 +165,7 @@ std::deque<HistoryLogModel::Message> HistoryLogModel::fetchData(uint64_t from_ti
     auto msgs = fetchData(first, events.cend(), 0);
     if (update_colors) {
       for (auto it = msgs.begin(); it != msgs.end(); ++it) {
-        hex_colors.compute(msg_id, it->data.data(), it->data.size(), it->mono_time / (double)1e9, speed, nullptr, freq);
+        hex_colors.compute(msg_id, it->data.data(), it->data.size(), it->mono_time / (double)1e9, speed, no_mask, freq);
         it->colors = hex_colors.colors;
       }
     }
@@ -177,7 +176,7 @@ std::deque<HistoryLogModel::Message> HistoryLogModel::fetchData(uint64_t from_ti
 // HeaderView
 
 QSize HeaderView::sectionSizeFromContents(int logicalIndex) const {
-  static QSize time_col_size = fontMetrics().boundingRect({0, 0, 200, 200}, defaultAlignment(), "000000.000").size() + QSize(10, 6);
+  static const QSize time_col_size = fontMetrics().boundingRect({0, 0, 200, 200}, defaultAlignment(), "000000.000").size() + QSize(10, 6);
   if (logicalIndex == 0) {
     return time_col_size;
   } else {
@@ -237,10 +236,11 @@ LogsWidget::LogsWidget(QWidget *parent) : QFrame(parent) {
   main_layout->addWidget(logs = new QTableView(this));
   logs->setModel(model = new HistoryLogModel(this));
   delegate = new MessageBytesDelegate(this);
-  logs->setItemDelegateForColumn(1, new MessageBytesDelegate(this));
   logs->setHorizontalHeader(new HeaderView(Qt::Horizontal, this));
   logs->horizontalHeader()->setDefaultAlignment(Qt::AlignRight | (Qt::Alignment)Qt::TextWordWrap);
   logs->horizontalHeader()->setSectionResizeMode(QHeaderView::ResizeToContents);
+  logs->verticalHeader()->setSectionResizeMode(QHeaderView::Fixed);
+  logs->verticalHeader()->setDefaultSectionSize(delegate->sizeForBytes(8).height());
   logs->verticalHeader()->setVisible(false);
   logs->setFrameShape(QFrame::NoFrame);
 

tools/cabana/historylog.h

@@ -46,9 +46,9 @@ public slots:
 public:
   struct Message {
     uint64_t mono_time = 0;
-    QVector<double> sig_values;
+    std::vector<double> sig_values;
-    QByteArray data;
+    std::vector<uint8_t> data;
-    QVector<QColor> colors;
+    std::vector<QColor> colors;
   };
 
   template <class InputIt>

tools/cabana/mainwin.cc

@@ -21,6 +21,7 @@
 #include "tools/cabana/commands.h"
 #include "tools/cabana/streamselector.h"
 #include "tools/cabana/tools/findsignal.h"
+#include "tools/replay/replay.h"
 
 MainWindow::MainWindow() : QMainWindow() {
   createDockWindows();
@@ -84,8 +85,8 @@ void MainWindow::createActions() {
   close_stream_act->setEnabled(false);
   file_menu->addSeparator();
 
-  file_menu->addAction(tr("New DBC File"), [this]() { newFile(); })->setShortcuts(QKeySequence::New);
+  file_menu->addAction(tr("New DBC File"), [this]() { newFile(); }, QKeySequence::New);
-  file_menu->addAction(tr("Open DBC File..."), [this]() { openFile(); })->setShortcuts(QKeySequence::Open);
+  file_menu->addAction(tr("Open DBC File..."), [this]() { openFile(); }, QKeySequence::Open);
 
   manage_dbcs_menu = file_menu->addMenu(tr("Manage &DBC Files"));
 
@@ -111,19 +112,15 @@ void MainWindow::createActions() {
   file_menu->addAction(tr("Load DBC From Clipboard"), [=]() { loadFromClipboard(); });
 
   file_menu->addSeparator();
-  save_dbc = file_menu->addAction(tr("Save DBC..."), this, &MainWindow::save);
+  save_dbc = file_menu->addAction(tr("Save DBC..."), this, &MainWindow::save, QKeySequence::Save);
-  save_dbc->setShortcuts(QKeySequence::Save);
+  save_dbc_as = file_menu->addAction(tr("Save DBC As..."), this, &MainWindow::saveAs, QKeySequence::SaveAs);
-
-  save_dbc_as = file_menu->addAction(tr("Save DBC As..."), this, &MainWindow::saveAs);
-  save_dbc_as->setShortcuts(QKeySequence::SaveAs);
-
   copy_dbc_to_clipboard = file_menu->addAction(tr("Copy DBC To Clipboard"), this, &MainWindow::saveToClipboard);
 
   file_menu->addSeparator();
-  file_menu->addAction(tr("Settings..."), this, &MainWindow::setOption)->setShortcuts(QKeySequence::Preferences);
+  file_menu->addAction(tr("Settings..."), this, &MainWindow::setOption, QKeySequence::Preferences);
 
   file_menu->addSeparator();
-  file_menu->addAction(tr("E&xit"), qApp, &QApplication::closeAllWindows)->setShortcuts(QKeySequence::Quit);
+  file_menu->addAction(tr("E&xit"), qApp, &QApplication::closeAllWindows, QKeySequence::Quit);
 
   // Edit Menu
   QMenu *edit_menu = menuBar()->addMenu(tr("&Edit"));
@@ -157,7 +154,7 @@ void MainWindow::createActions() {
 
   // Help Menu
   QMenu *help_menu = menuBar()->addMenu(tr("&Help"));
-  help_menu->addAction(tr("Help"), this, &MainWindow::onlineHelp)->setShortcuts(QKeySequence::HelpContents);
+  help_menu->addAction(tr("Help"), this, &MainWindow::onlineHelp, QKeySequence::HelpContents);
   help_menu->addAction(tr("About &Qt"), qApp, &QApplication::aboutQt);
 }
 
@@ -374,7 +371,7 @@ void MainWindow::eventsMerged() {
       auto dbc_name = fingerprint_to_dbc[car_fingerprint];
       if (dbc_name != QJsonValue::Undefined) {
         // Prevent dialog that load autosaved file from blocking replay->start().
-        QTimer::singleShot(0, [dbc_name, this]() { loadDBCFromOpendbc(dbc_name.toString()); });
+        QTimer::singleShot(0, this, [dbc_name, this]() { loadDBCFromOpendbc(dbc_name.toString()); });
       }
     }
   }
@@ -471,11 +468,7 @@ void MainWindow::saveFileToClipboard(DBCFile *dbc_file) {
 
 void MainWindow::updateLoadSaveMenus() {
   int cnt = dbc()->nonEmptyDBCCount();
-  if (cnt > 1) {
+  save_dbc->setText(cnt > 1 ? tr("Save %1 DBCs...").arg(cnt) : tr("Save DBC..."));
-    save_dbc->setText(tr("Save %1 DBCs...").arg(dbc()->dbcCount()));
-  } else {
-    save_dbc->setText(tr("Save DBC..."));
-  }
   save_dbc->setEnabled(cnt > 0);
   save_dbc_as->setEnabled(cnt == 1);
 
@@ -608,12 +601,6 @@ void MainWindow::closeEvent(QCloseEvent *event) {
   }
   settings.message_header_state = messages_widget->saveHeaderState();
 
-  auto status = settings.save();
-  if (status == QSettings::AccessError) {
-    QString error = tr("Failed to write settings to [%1]: access denied").arg(Settings::filePath());
-    qDebug() << error;
-    QMessageBox::warning(this, tr("Failed to write settings"), error);
-  }
   QWidget::closeEvent(event);
 }
 

tools/cabana/messageswidget.cc

@@ -1,10 +1,8 @@
 #include "tools/cabana/messageswidget.h"
 
-#include <algorithm>
 #include <limits>
-#include <utility>
-#include <vector>
 
+#include <QCheckBox>
 #include <QHBoxLayout>
 #include <QPainter>
 #include <QPushButton>
@@ -13,34 +11,17 @@
 
 #include "tools/cabana/commands.h"
 
-MessagesWidget::MessagesWidget(QWidget *parent) : QWidget(parent) {
+MessagesWidget::MessagesWidget(QWidget *parent) : menu(new QMenu(this)), QWidget(parent) {
   QVBoxLayout *main_layout = new QVBoxLayout(this);
   main_layout->setContentsMargins(0, 0, 0, 0);
-
+  main_layout->setSpacing(0);
-  QHBoxLayout *title_layout = new QHBoxLayout();
+  // toolbar
-  num_msg_label = new QLabel(this);
+  main_layout->addWidget(createToolBar());
-  title_layout->addSpacing(10);
-  title_layout->addWidget(num_msg_label);
-
-  title_layout->addStretch();
-  title_layout->addWidget(multiple_lines_bytes = new QCheckBox(tr("Multiple Lines &Bytes"), this));
-  multiple_lines_bytes->setToolTip(tr("Display bytes in multiple lines"));
-  multiple_lines_bytes->setChecked(settings.multiple_lines_bytes);
-  QPushButton *clear_filters = new QPushButton(tr("&Clear Filters"));
-  clear_filters->setEnabled(false);
-  title_layout->addWidget(clear_filters);
-  main_layout->addLayout(title_layout);
-
   // message table
-  view = new MessageView(this);
+  main_layout->addWidget(view = new MessageView(this));
-  model = new MessageListModel(this);
+  view->setItemDelegate(delegate = new MessageBytesDelegate(view, settings.multiple_lines_hex));
-  header = new MessageViewHeader(this);
+  view->setModel(model = new MessageListModel(this));
-  auto delegate = new MessageBytesDelegate(view, settings.multiple_lines_bytes);
+  view->setHeader(header = new MessageViewHeader(this));
-
-  view->setItemDelegate(delegate);
-  view->setHeader(header);
-  view->setModel(model);
-  view->setHeader(header);
   view->setSortingEnabled(true);
   view->sortByColumn(MessageListModel::Column::NAME, Qt::AscendingOrder);
   view->setAllColumnsShowFocus(true);
@@ -48,49 +29,33 @@ MessagesWidget::MessagesWidget(QWidget *parent) : QWidget(parent) {
   view->setItemsExpandable(false);
   view->setIndentation(0);
   view->setRootIsDecorated(false);
+  view->setUniformRowHeights(!settings.multiple_lines_hex);
 
   // Must be called before setting any header parameters to avoid overriding
   restoreHeaderState(settings.message_header_state);
-  view->header()->setSectionsMovable(true);
+  header->setSectionsMovable(true);
-  view->header()->setSectionResizeMode(MessageListModel::Column::DATA, QHeaderView::Fixed);
+  header->setSectionResizeMode(MessageListModel::Column::DATA, QHeaderView::Fixed);
-  view->header()->setStretchLastSection(true);
+  header->setStretchLastSection(true);
-
+  header->setContextMenuPolicy(Qt::CustomContextMenu);
-  // Header context menu
-  view->header()->setContextMenuPolicy(Qt::CustomContextMenu);
-  QObject::connect(view->header(), &QHeaderView::customContextMenuRequested, view, &MessageView::headerContextMenuEvent);
-
-  main_layout->addWidget(view);
 
   // suppress
   QHBoxLayout *suppress_layout = new QHBoxLayout();
-  suppress_add = new QPushButton("Suppress Highlighted");
+  suppress_layout->addWidget(suppress_add = new QPushButton("Suppress Highlighted"));
-  suppress_clear = new QPushButton();
+  suppress_layout->addWidget(suppress_clear = new QPushButton());
-  suppress_layout->addWidget(suppress_add);
+  suppress_clear->setToolTip(tr("Clear suppressed"));
-  suppress_layout->addWidget(suppress_clear);
+  suppress_layout->addStretch(1);
-  QCheckBox *suppress_defined_signals = new QCheckBox(tr("Suppress Defined Signals"), this);
+  QCheckBox *suppress_defined_signals = new QCheckBox(tr("Suppress Signals"), this);
+  suppress_defined_signals->setToolTip(tr("Suppress defined signals"));
   suppress_defined_signals->setChecked(settings.suppress_defined_signals);
   suppress_layout->addWidget(suppress_defined_signals);
   main_layout->addLayout(suppress_layout);
 
   // signals/slots
+  QObject::connect(menu, &QMenu::aboutToShow, this, &MessagesWidget::menuAboutToShow);
   QObject::connect(header, &MessageViewHeader::filtersUpdated, model, &MessageListModel::setFilterStrings);
-  QObject::connect(header, &MessageViewHeader::filtersUpdated, [=](const QMap<int, QString> &filters) {
+  QObject::connect(header, &MessageViewHeader::customContextMenuRequested, this, &MessagesWidget::headerContextMenuEvent);
-    clear_filters->setEnabled(!filters.isEmpty());
-  });
   QObject::connect(view->horizontalScrollBar(), &QScrollBar::valueChanged, header, &MessageViewHeader::updateHeaderPositions);
-  QObject::connect(clear_filters, &QPushButton::clicked, header, &MessageViewHeader::clearFilters);
+  QObject::connect(suppress_defined_signals, &QCheckBox::stateChanged, can, &AbstractStream::suppressDefinedSignals);
-  QObject::connect(multiple_lines_bytes, &QCheckBox::stateChanged, [=](int state) {
-    settings.multiple_lines_bytes = (state == Qt::Checked);
-    delegate->setMultipleLines(settings.multiple_lines_bytes);
-    view->setUniformRowHeights(!settings.multiple_lines_bytes);
-
-    // Reset model to force recalculation of the width of the bytes column
-    model->forceResetModel();
-  });
-  QObject::connect(suppress_defined_signals, &QCheckBox::stateChanged, [=](int state) {
-    settings.suppress_defined_signals = (state == Qt::Checked);
-    emit settings.changed();
-  });
   QObject::connect(can, &AbstractStream::msgsReceived, model, &MessageListModel::msgsReceived);
   QObject::connect(dbc(), &DBCManager::DBCFileChanged, this, &MessagesWidget::dbcModified);
   QObject::connect(UndoStack::instance(), &QUndoStack::indexChanged, this, &MessagesWidget::dbcModified);
@@ -101,24 +66,17 @@ MessagesWidget::MessagesWidget(QWidget *parent) : QWidget(parent) {
     view->updateBytesSectionSize();
   });
   QObject::connect(view->selectionModel(), &QItemSelectionModel::currentChanged, [=](const QModelIndex &current, const QModelIndex &previous) {
-    if (current.isValid() && current.row() < model->msgs.size()) {
+    if (current.isValid() && current.row() < model->items_.size()) {
-      auto &id = model->msgs[current.row()];
+      const auto &id = model->items_[current.row()].id;
       if (!current_msg_id || id != *current_msg_id) {
         current_msg_id = id;
         emit msgSelectionChanged(*current_msg_id);
       }
     }
   });
-  QObject::connect(suppress_add, &QPushButton::clicked, [=]() {
+  QObject::connect(suppress_add, &QPushButton::clicked, this, &MessagesWidget::suppressHighlighted);
-    model->suppress();
+  QObject::connect(suppress_clear, &QPushButton::clicked, this, &MessagesWidget::suppressHighlighted);
-    updateSuppressedButtons();
+  suppressHighlighted();
-  });
-  QObject::connect(suppress_clear, &QPushButton::clicked, [=]() {
-    model->clearSuppress();
-    updateSuppressedButtons();
-  });
-
-  updateSuppressedButtons();
 
   setWhatsThis(tr(R"(
     <b>Message View</b><br/>
@@ -130,26 +88,73 @@ MessagesWidget::MessagesWidget(QWidget *parent) : QWidget(parent) {
   )"));
 }
 
+QToolBar *MessagesWidget::createToolBar() {
+  QToolBar *toolbar = new QToolBar(this);
+  toolbar->setIconSize({12, 12});
+  toolbar->addWidget(num_msg_label = new QLabel(this));
+  num_msg_label->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Preferred);
+
+  auto views_btn = toolbar->addAction(utils::icon("three-dots"), tr("View..."));
+  views_btn->setMenu(menu);
+  auto view_button = qobject_cast<QToolButton *>(toolbar->widgetForAction(views_btn));
+  view_button->setPopupMode(QToolButton::InstantPopup);
+  view_button->setToolButtonStyle(Qt::ToolButtonIconOnly);
+  view_button->setStyleSheet("QToolButton::menu-indicator { image: none; }");
+  return toolbar;
+}
+
 void MessagesWidget::dbcModified() {
   num_msg_label->setText(tr("%1 Messages, %2 Signals").arg(dbc()->msgCount()).arg(dbc()->signalCount()));
   model->dbcModified();
 }
 
 void MessagesWidget::selectMessage(const MessageId &msg_id) {
-  auto it = std::find(model->msgs.cbegin(), model->msgs.cend(), msg_id);
+  auto it = std::find_if(model->items_.cbegin(), model->items_.cend(),
-  if (it != model->msgs.cend()) {
+                         [&msg_id](auto &item) { return item.id == msg_id; });
-    view->setCurrentIndex(model->index(std::distance(model->msgs.cbegin(), it), 0));
+  if (it != model->items_.cend()) {
+    view->setCurrentIndex(model->index(std::distance(model->items_.cbegin(), it), 0));
   }
 }
 
-void MessagesWidget::updateSuppressedButtons() {
+void MessagesWidget::suppressHighlighted() {
-  if (model->suppressed_bytes.empty()) {
+  if (sender() == suppress_add) {
-    suppress_clear->setEnabled(false);
+    size_t n = can->suppressHighlighted();
-    suppress_clear->setText("Clear Suppressed");
+    suppress_clear->setText(tr("Clear (%1)").arg(n));
-  } else {
     suppress_clear->setEnabled(true);
-    suppress_clear->setText(QString("Clear Suppressed (%1)").arg(model->suppressed_bytes.size()));
+  } else {
+    can->clearSuppressed();
+    suppress_clear->setText(tr("Clear"));
+    suppress_clear->setEnabled(false);
+  }
+}
+
+void MessagesWidget::headerContextMenuEvent(const QPoint &pos) {
+  menu->exec(header->mapToGlobal(pos));
+}
+
+void MessagesWidget::menuAboutToShow() {
+  menu->clear();
+  for (int i = 0; i < header->count(); ++i) {
+    int logical_index = header->logicalIndex(i);
+    auto action = menu->addAction(model->headerData(logical_index, Qt::Horizontal).toString(),
+                                  [=](bool checked) { header->setSectionHidden(logical_index, !checked); });
+    action->setCheckable(true);
+    action->setChecked(!header->isSectionHidden(logical_index));
+    // Can't hide the name column
+    action->setEnabled(logical_index > 0);
   }
+  menu->addSeparator();
+  auto action = menu->addAction(tr("Mutlti-Line bytes"), this, &MessagesWidget::setMultiLineBytes);
+  action->setCheckable(true);
+  action->setChecked(settings.multiple_lines_hex);
+}
+
+void MessagesWidget::setMultiLineBytes(bool multi) {
+  settings.multiple_lines_hex = multi;
+  delegate->setMultipleLines(multi);
+  view->setUniformRowHeights(!multi);
+  view->updateBytesSectionSize();
+  view->doItemsLayout();
 }
 
 // MessageListModel
@@ -160,6 +165,7 @@ QVariant MessageListModel::headerData(int section, Qt::Orientation orientation,
       case Column::NAME: return tr("Name");
       case Column::SOURCE: return tr("Bus");
       case Column::ADDRESS: return tr("ID");
+      case Column::NODE: return tr("Node");
       case Column::FREQ: return tr("Freq");
       case Column::COUNT: return tr("Count");
       case Column::DATA: return tr("Bytes");
@@ -169,43 +175,34 @@ QVariant MessageListModel::headerData(int section, Qt::Orientation orientation,
 }
 
 QVariant MessageListModel::data(const QModelIndex &index, int role) const {
-  if (!index.isValid() || index.row() >= msgs.size()) return {};
+  if (!index.isValid() || index.row() >= items_.size()) return {};
-
-  const auto &id = msgs[index.row()];
-  auto &can_data = can->lastMessage(id);
 
-  auto getFreq = [](const CanData &d) -> QString {
+  auto getFreq = [](const CanData &d) {
     if (d.freq > 0 && (can->currentSec() - d.ts - 1.0 / settings.fps) < (5.0 / d.freq)) {
       return d.freq >= 0.95 ? QString::number(std::nearbyint(d.freq)) : QString::number(d.freq, 'f', 2);
     } else {
-      return "--";
+      return QStringLiteral("--");
     }
   };
 
+  const auto &item = items_[index.row()];
   if (role == Qt::DisplayRole) {
     switch (index.column()) {
-      case Column::NAME: return msgName(id);
+      case Column::NAME: return item.name;
-      case Column::SOURCE: return id.source != INVALID_SOURCE ? QString::number(id.source) : "N/A";
+      case Column::SOURCE: return item.id.source != INVALID_SOURCE ? QString::number(item.id.source) : "N/A";
-      case Column::ADDRESS: return QString::number(id.address, 16);
+      case Column::ADDRESS: return QString::number(item.id.address, 16);
-      case Column::FREQ: return id.source != INVALID_SOURCE ? getFreq(can_data) : "N/A";
+      case Column::NODE: return item.node;
-      case Column::COUNT: return id.source != INVALID_SOURCE ? QString::number(can_data.count) : "N/A";
+      case Column::FREQ: return item.id.source != INVALID_SOURCE ? getFreq(*item.data) : "N/A";
-      case Column::DATA: return id.source != INVALID_SOURCE ? toHex(can_data.dat) : "N/A";
+      case Column::COUNT: return item.id.source != INVALID_SOURCE ? QString::number(item.data->count) : "N/A";
+      case Column::DATA: return item.id.source != INVALID_SOURCE ? "" : "N/A";
     }
   } else if (role == ColorsRole) {
-    QVector<QColor> colors = can_data.colors;
+    return QVariant::fromValue((void*)(&item.data->colors));
-    if (!suppressed_bytes.empty()) {
+  } else if (role == BytesRole && index.column() == Column::DATA && item.id.source != INVALID_SOURCE) {
-      for (int i = 0; i < colors.size(); i++) {
+    return QVariant::fromValue((void*)(&item.data->dat));
-        if (suppressed_bytes.contains({id, i})) {
-          colors[i] = QColor(255, 255, 255, 0);
-        }
-      }
-    }
-    return QVariant::fromValue(colors);
-  } else if (role == BytesRole && index.column() == Column::DATA && id.source != INVALID_SOURCE) {
-    return can_data.dat;
   } else if (role == Qt::ToolTipRole && index.column() == Column::NAME) {
-    auto msg = dbc()->msg(id);
+    auto msg = dbc()->msg(item.id);
-    auto tooltip = msg ? msg->name : UNTITLED;
+    auto tooltip = item.name;
     if (msg && !msg->comment.isEmpty()) tooltip += "<br /><span style=\"color:gray;\">" + msg->comment + "</span>";
     return tooltip;
   }
@@ -213,49 +210,31 @@ QVariant MessageListModel::data(const QModelIndex &index, int role) const {
 }
 
 void MessageListModel::setFilterStrings(const QMap<int, QString> &filters) {
-  filter_str = filters;
+  filters_ = filters;
-  fetchData();
+  filterAndSort();
 }
 
 void MessageListModel::dbcModified() {
-  dbc_address.clear();
+  dbc_messages_.clear();
   for (const auto &[_, m] : dbc()->getMessages(-1)) {
-    dbc_address.insert(m.address);
+    dbc_messages_.insert(MessageId{.source = INVALID_SOURCE, .address = m.address});
   }
-  fetchData();
+  filterAndSort(true);
 }
 
-void MessageListModel::sortMessages(std::vector<MessageId> &new_msgs) {
+void MessageListModel::sortItems(std::vector<MessageListModel::Item> &items) {
-  if (sort_column == Column::NAME) {
+  auto do_sort = [order = sort_order](std::vector<MessageListModel::Item> &m, auto proj) {
-    std::sort(new_msgs.begin(), new_msgs.end(), [=](auto &l, auto &r) {
+    std::stable_sort(m.begin(), m.end(), [order, proj = std::move(proj)](auto &l, auto &r) {
-      auto ll = std::pair{msgName(l), l};
+      return order == Qt::AscendingOrder ? proj(l) < proj(r) : proj(l) > proj(r);
-      auto rr = std::pair{msgName(r), r};
-      return sort_order == Qt::AscendingOrder ? ll < rr : ll > rr;
-    });
-  } else if (sort_column == Column::SOURCE) {
-    std::sort(new_msgs.begin(), new_msgs.end(), [=](auto &l, auto &r) {
-      auto ll = std::pair{l.source, l};
-      auto rr = std::pair{r.source, r};
-      return sort_order == Qt::AscendingOrder ? ll < rr : ll > rr;
-    });
-  } else if (sort_column == Column::ADDRESS) {
-    std::sort(new_msgs.begin(), new_msgs.end(), [=](auto &l, auto &r) {
-      auto ll = std::pair{l.address, l};
-      auto rr = std::pair{r.address, r};
-      return sort_order == Qt::AscendingOrder ? ll < rr : ll > rr;
-    });
-  } else if (sort_column == Column::FREQ) {
-    std::sort(new_msgs.begin(), new_msgs.end(), [=](auto &l, auto &r) {
-      auto ll = std::pair{can->lastMessage(l).freq, l};
-      auto rr = std::pair{can->lastMessage(r).freq, r};
-      return sort_order == Qt::AscendingOrder ? ll < rr : ll > rr;
-    });
-  } else if (sort_column == Column::COUNT) {
-    std::sort(new_msgs.begin(), new_msgs.end(), [=](auto &l, auto &r) {
-      auto ll = std::pair{can->lastMessage(l).count, l};
-      auto rr = std::pair{can->lastMessage(r).count, r};
-      return sort_order == Qt::AscendingOrder ? ll < rr : ll > rr;
     });
+  };
+  switch (sort_column) {
+    case Column::NAME: do_sort(items, [](auto &item) { return std::tie(item.name, item.id); }); break;
+    case Column::SOURCE: do_sort(items, [](auto &item) { return std::tie(item.id.source, item.id); }); break;
+    case Column::ADDRESS: do_sort(items, [](auto &item) { return std::tie(item.id.address, item.id);}); break;
+    case Column::NODE: do_sort(items, [](auto &item) { return std::tie(item.node, item.id);}); break;
+    case Column::FREQ: do_sort(items, [](auto &item) { return std::tie(item.data->freq, item.id); }); break;
+    case Column::COUNT: do_sort(items, [](auto &item) { return std::tie(item.data->count, item.id); }); break;
   }
 }
 
@@ -274,80 +253,85 @@ static bool parseRange(const QString &filter, uint32_t value, int base = 10) {
   return ok && value >= min && value <= max;
 }
 
-bool MessageListModel::matchMessage(const MessageId &id, const CanData &data, const QMap<int, QString> &filters) {
+bool MessageListModel::match(const MessageListModel::Item &item) {
+  if (filters_.isEmpty())
+    return true;
+
   bool match = true;
-  for (auto it = filters.cbegin(); it != filters.cend() && match; ++it) {
+  for (auto it = filters_.cbegin(); it != filters_.cend() && match; ++it) {
     const QString &txt = it.value();
-    QRegularExpression re(txt, QRegularExpression::CaseInsensitiveOption | QRegularExpression::DotMatchesEverythingOption);
     switch (it.key()) {
       case Column::NAME: {
-        const auto msg = dbc()->msg(id);
+        match = item.name.contains(txt, Qt::CaseInsensitive);
-        match = re.match(msg ? msg->name : UNTITLED).hasMatch();
+        if (!match) {
-        match = match || (msg && std::any_of(msg->sigs.cbegin(), msg->sigs.cend(),
+          const auto m = dbc()->msg(item.id);
-                                             [&re](const auto &s) { return re.match(s->name).hasMatch(); }));
+          match = m && std::any_of(m->sigs.cbegin(), m->sigs.cend(),
+                                   [&txt](const auto &s) { return s->name.contains(txt, Qt::CaseInsensitive); });
+        }
         break;
       }
       case Column::SOURCE:
-        match = parseRange(txt, id.source);
+        match = parseRange(txt, item.id.source);
         break;
-      case Column::ADDRESS: {
+      case Column::ADDRESS:
-        match = re.match(QString::number(id.address, 16)).hasMatch();
+        match = QString::number(item.id.address, 16).contains(txt, Qt::CaseInsensitive);
-        match = match || parseRange(txt, id.address, 16);
+        match = match || parseRange(txt, item.id.address, 16);
+        break;
+      case Column::NODE:
+        match = item.node.contains(txt, Qt::CaseInsensitive);
         break;
-      }
       case Column::FREQ:
         // TODO: Hide stale messages?
-        match = parseRange(txt, data.freq);
+        match = parseRange(txt, item.data->freq);
         break;
       case Column::COUNT:
-        match = parseRange(txt, data.count);
+        match = parseRange(txt, item.data->count);
         break;
-      case Column::DATA: {
+      case Column::DATA:
-        match = QString(data.dat.toHex()).contains(txt, Qt::CaseInsensitive);
+        match = utils::toHex(item.data->dat).contains(txt, Qt::CaseInsensitive);
-        match = match || re.match(QString(data.dat.toHex())).hasMatch();
-        match = match || re.match(QString(data.dat.toHex(' '))).hasMatch();
         break;
-      }
     }
   }
   return match;
 }
 
-void MessageListModel::fetchData() {
+void MessageListModel::filterAndSort(bool force_reset) {
-  std::vector<MessageId> new_msgs;
+  // merge CAN and DBC messages
-  new_msgs.reserve(can->last_msgs.size() + dbc_address.size());
+  std::vector<MessageId> all_messages;
-
+  all_messages.reserve(can->lastMessages().size() + dbc_messages_.size());
-  auto address = dbc_address;
+  auto dbc_msgs = dbc_messages_;
-  for (auto it = can->last_msgs.cbegin(); it != can->last_msgs.cend(); ++it) {
+  for (const auto &[id, m] : can->lastMessages()) {
-    if (filter_str.isEmpty() || matchMessage(it.key(), it.value(), filter_str)) {
+    all_messages.push_back(id);
-      new_msgs.push_back(it.key());
+    dbc_msgs.erase(MessageId{.source = INVALID_SOURCE, .address = id.address});
-    }
-    address.remove(it.key().address);
   }
+  std::copy(dbc_msgs.begin(), dbc_msgs.end(), std::back_inserter(all_messages));
 
-  // merge all DBC messages
+  // filter and sort
-  for (auto &addr : address) {
+  std::vector<Item> items;
-    MessageId id{.source = INVALID_SOURCE, .address = addr};
+  for (const auto &id : all_messages) {
-    if (filter_str.isEmpty() || matchMessage(id, {}, filter_str)) {
+    auto msg = dbc()->msg(id);
-      new_msgs.push_back(id);
+    Item item = {.id = id,
-    }
+                 .name = msg ? msg->name : UNTITLED,
+                 .node = msg ? msg->transmitter : QString(),
+                 .data = &can->lastMessage(id)};
+    if (match(item))
+      items.emplace_back(item);
   }
+  sortItems(items);
 
-  sortMessages(new_msgs);
+  if (force_reset || items_ != items) {
-
-  if (msgs != new_msgs) {
     beginResetModel();
-    msgs = std::move(new_msgs);
+    items_ = std::move(items);
     endResetModel();
   }
 }
 
-void MessageListModel::msgsReceived(const QHash<MessageId, CanData> *new_msgs, bool has_new_ids) {
+void MessageListModel::msgsReceived(const std::set<MessageId> *new_msgs, bool has_new_ids) {
-  if (has_new_ids || filter_str.contains(Column::FREQ) || filter_str.contains(Column::COUNT) || filter_str.contains(Column::DATA)) {
+  if (has_new_ids || filters_.contains(Column::FREQ) || filters_.contains(Column::COUNT) || filters_.contains(Column::DATA)) {
-    fetchData();
+    filterAndSort();
   }
-  for (int i = 0; i < msgs.size(); ++i) {
+  for (int i = 0; i < items_.size(); ++i) {
-    if (new_msgs->contains(msgs[i])) {
+    if (!new_msgs || new_msgs->count(items_[i].id)) {
       for (int col = Column::FREQ; col < columnCount(); ++col)
         emit dataChanged(index(i, col), index(i, col), {Qt::DisplayRole});
     }
@@ -355,36 +339,13 @@ void MessageListModel::msgsReceived(const QHash<MessageId, CanData> *new_msgs, b
 }
 
 void MessageListModel::sort(int column, Qt::SortOrder order) {
-  if (column != columnCount() - 1) {
+  if (column != Column::DATA) {
     sort_column = column;
     sort_order = order;
-    fetchData();
+    filterAndSort();
   }
 }
 
-void MessageListModel::suppress() {
-  const double cur_ts = can->currentSec();
-
-  for (auto &id : msgs) {
-    auto &can_data = can->lastMessage(id);
-    for (int i = 0; i < can_data.dat.size(); i++) {
-      const double dt = cur_ts - can_data.last_change_t[i];
-      if (dt < 2.0) {
-        suppressed_bytes.insert({id, i});
-      }
-    }
-  }
-}
-
-void MessageListModel::clearSuppress() {
-  suppressed_bytes.clear();
-}
-
-void MessageListModel::forceResetModel() {
-  beginResetModel();
-  endResetModel();
-}
-
 // MessageView
 
 void MessageView::drawRow(QPainter *painter, const QStyleOptionViewItem &option, const QModelIndex &index) const {
@@ -415,45 +376,14 @@ void MessageView::updateBytesSectionSize() {
   auto delegate = ((MessageBytesDelegate *)itemDelegate());
   int max_bytes = 8;
   if (!delegate->multipleLines()) {
-    for (auto it = can->last_msgs.constBegin(); it != can->last_msgs.constEnd(); ++it) {
+    for (const auto &[_, m] : can->lastMessages()) {
-      max_bytes = std::max(max_bytes, it.value().dat.size());
+      max_bytes = std::max<int>(max_bytes, m.dat.size());
     }
   }
-  int width = delegate->widthForBytes(max_bytes);
+  header()->resizeSection(MessageListModel::Column::DATA, delegate->sizeForBytes(max_bytes).width());
-  if (header()->sectionSize(MessageListModel::Column::DATA) != width) {
-    header()->resizeSection(MessageListModel::Column::DATA, width);
-  }
 }
 
-void MessageView::headerContextMenuEvent(const QPoint &pos) {
+// MessageViewHeader
-  QMenu menu(this);
-  int cur_index = header()->logicalIndexAt(pos);
-
-  QAction *action;
-  for (int visual_index = 0; visual_index < header()->count(); visual_index++) {
-    int logical_index = header()->logicalIndex(visual_index);
-    QString column_name = model()->headerData(logical_index, Qt::Horizontal).toString();
-
-    // Hide show action
-    if (header()->isSectionHidden(logical_index)) {
-      action = menu.addAction(tr("  %1").arg(column_name), [=]() { header()->showSection(logical_index); });
-    } else {
-      action = menu.addAction(tr("✓ %1").arg(column_name), [=]() { header()->hideSection(logical_index); });
-    }
-
-    // Can't hide the name column
-    action->setEnabled(logical_index > 0);
-
-    // Make current column bold
-    if (logical_index == cur_index) {
-      QFont font = action->font();
-      font.setBold(true);
-      action->setFont(font);
-    }
-  }
-
-  menu.exec(header()->mapToGlobal(pos));
-}
 
 MessageViewHeader::MessageViewHeader(QWidget *parent) : QHeaderView(Qt::Horizontal, parent) {
   QObject::connect(this, &QHeaderView::sectionResized, this, &MessageViewHeader::updateHeaderPositions);
@@ -463,29 +393,19 @@ MessageViewHeader::MessageViewHeader(QWidget *parent) : QHeaderView(Qt::Horizont
 void MessageViewHeader::updateFilters() {
   QMap<int, QString> filters;
   for (int i = 0; i < count(); i++) {
-    if (editors[i]) {
+    if (editors[i] && !editors[i]->text().isEmpty()) {
-      QString filter = editors[i]->text();
+      filters[i] = editors[i]->text();
-      if (!filter.isEmpty()) {
-        filters[i] = filter;
-      }
     }
   }
   emit filtersUpdated(filters);
 }
 
-void MessageViewHeader::clearFilters() {
-  for (QLineEdit *editor : editors) {
-    editor->clear();
-  }
-}
-
 void MessageViewHeader::updateHeaderPositions() {
   QSize sz = QHeaderView::sizeHint();
   for (int i = 0; i < count(); i++) {
     if (editors[i]) {
       int h = editors[i]->sizeHint().height();
-      editors[i]->move(sectionViewportPosition(i), sz.height());
+      editors[i]->setGeometry(sectionViewportPosition(i), sz.height(), sectionSize(i), h);
-      editors[i]->resize(sectionSize(i), h);
       editors[i]->setHidden(isSectionHidden(i));
     }
   }
@@ -508,11 +428,7 @@ void MessageViewHeader::updateGeometries() {
   updateHeaderPositions();
 }
 
-
 QSize MessageViewHeader::sizeHint() const {
   QSize sz = QHeaderView::sizeHint();
-  if (editors[0]) {
+  return editors[0] ? QSize(sz.width(), sz.height() + editors[0]->height() + 1) : sz;
-    sz.setHeight(sz.height() + editors[0]->minimumSizeHint().height() + 1);
-  }
-  return sz;
 }

tools/cabana/messageswidget.h

@@ -1,17 +1,16 @@
 #pragma once
 
 #include <algorithm>
+#include <set>
 #include <utility>
 #include <vector>
 
 #include <QAbstractTableModel>
-#include <QCheckBox>
-#include <QContextMenuEvent>
 #include <QHeaderView>
 #include <QLabel>
 #include <QLineEdit>
 #include <QMenu>
-#include <QSet>
+#include <QToolBar>
 #include <QTreeView>
 
 #include "tools/cabana/dbc/dbcmanager.h"
@@ -21,11 +20,11 @@ class MessageListModel : public QAbstractTableModel {
 Q_OBJECT
 
 public:
-
   enum Column {
     NAME = 0,
     SOURCE,
     ADDRESS,
+    NODE,
     FREQ,
     COUNT,
     DATA,
@@ -35,24 +34,28 @@ public:
   QVariant headerData(int section, Qt::Orientation orientation, int role = Qt::DisplayRole) const override;
   int columnCount(const QModelIndex &parent = QModelIndex()) const override { return Column::DATA + 1; }
   QVariant data(const QModelIndex &index, int role = Qt::DisplayRole) const;
-  int rowCount(const QModelIndex &parent = QModelIndex()) const override { return msgs.size(); }
+  int rowCount(const QModelIndex &parent = QModelIndex()) const override { return items_.size(); }
   void sort(int column, Qt::SortOrder order = Qt::AscendingOrder) override;
   void setFilterStrings(const QMap<int, QString> &filters);
-  void msgsReceived(const QHash<MessageId, CanData> *new_msgs, bool has_new_ids);
+  void msgsReceived(const std::set<MessageId> *new_msgs, bool has_new_ids);
-  void fetchData();
+  void filterAndSort(bool force_reset = false);
-  void suppress();
-  void clearSuppress();
-  void forceResetModel();
   void dbcModified();
-  std::vector<MessageId> msgs;
+
-  QSet<std::pair<MessageId, int>> suppressed_bytes;
+  struct Item {
+    MessageId id;
+    QString name;
+    QString node;
+    const CanData *data;
+    bool operator==(const Item &other) const { return id == other.id; }
+  };
+  std::vector<Item> items_;
 
 private:
-  void sortMessages(std::vector<MessageId> &new_msgs);
+  void sortItems(std::vector<MessageListModel::Item> &items);
-  bool matchMessage(const MessageId &id, const CanData &data, const QMap<int, QString> &filters);
+  bool match(const MessageListModel::Item &id);
 
-  QMap<int, QString> filter_str;
+  QMap<int, QString> filters_;
-  QSet<uint32_t> dbc_address;
+  std::set<MessageId> dbc_messages_;
   int sort_column = 0;
   Qt::SortOrder sort_order = Qt::AscendingOrder;
 };
@@ -65,23 +68,17 @@ public:
   void drawBranches(QPainter *painter, const QRect &rect, const QModelIndex &index) const override {}
   void dataChanged(const QModelIndex &topLeft, const QModelIndex &bottomRight, const QVector<int> &roles = QVector<int>()) override;
   void updateBytesSectionSize();
-  void headerContextMenuEvent(const QPoint &pos);
 };
 
 class MessageViewHeader : public QHeaderView {
   // https://stackoverflow.com/a/44346317
-
   Q_OBJECT
 public:
   MessageViewHeader(QWidget *parent);
   void updateHeaderPositions();
-
   void updateGeometries() override;
   QSize sizeHint() const override;
 
-public slots:
-  void clearFilters();
-
 signals:
   void filtersUpdated(const QMap<int, QString> &filters);
 
@@ -99,7 +96,7 @@ public:
   void selectMessage(const MessageId &message_id);
   QByteArray saveHeaderState() const { return view->header()->saveState(); }
   bool restoreHeaderState(const QByteArray &state) const { return view->header()->restoreState(state); }
-  void updateSuppressedButtons();
+  void suppressHighlighted();
 
 public slots:
   void dbcModified();
@@ -108,12 +105,18 @@ signals:
   void msgSelectionChanged(const MessageId &message_id);
 
 protected:
+  QToolBar *createToolBar();
+  void headerContextMenuEvent(const QPoint &pos);
+  void menuAboutToShow();
+  void setMultiLineBytes(bool multi);
+
   MessageView *view;
   MessageViewHeader *header;
+  MessageBytesDelegate *delegate;
   std::optional<MessageId> current_msg_id;
-  QCheckBox *multiple_lines_bytes;
   MessageListModel *model;
   QPushButton *suppress_add;
   QPushButton *suppress_clear;
   QLabel *num_msg_label;
+  QMenu *menu;
 };

tools/cabana/settings.cc

@@ -6,63 +6,51 @@
 #include <QFileDialog>
 #include <QFormLayout>
 #include <QPushButton>
+#include <QSettings>
 #include <QStandardPaths>
+#include <type_traits>
 
 #include "tools/cabana/util.h"
 
 Settings settings;
 
-QSettings::Status Settings::save() {
+template <class SettingOperation>
-  QSettings s(filePath(), QSettings::IniFormat);
+void settings_op(SettingOperation op) {
-  s.setValue("fps", fps);
+  QSettings s("cabana");
-  s.setValue("max_cached_minutes", max_cached_minutes);
+  op(s, "absolute_time", settings.absolute_time);
-  s.setValue("chart_height", chart_height);
+  op(s, "fps", settings.fps);
-  s.setValue("chart_range", chart_range);
+  op(s, "max_cached_minutes", settings.max_cached_minutes);
-  s.setValue("chart_column_count", chart_column_count);
+  op(s, "chart_height", settings.chart_height);
-  s.setValue("last_dir", last_dir);
+  op(s, "chart_range", settings.chart_range);
-  s.setValue("last_route_dir", last_route_dir);
+  op(s, "chart_column_count", settings.chart_column_count);
-  s.setValue("window_state", window_state);
+  op(s, "last_dir", settings.last_dir);
-  s.setValue("geometry", geometry);
+  op(s, "last_route_dir", settings.last_route_dir);
-  s.setValue("video_splitter_state", video_splitter_state);
+  op(s, "window_state", settings.window_state);
-  s.setValue("recent_files", recent_files);
+  op(s, "geometry", settings.geometry);
-  s.setValue("message_header_state_v3", message_header_state);
+  op(s, "video_splitter_state", settings.video_splitter_state);
-  s.setValue("chart_series_type", chart_series_type);
+  op(s, "recent_files", settings.recent_files);
-  s.setValue("theme", theme);
+  op(s, "message_header_state", settings.message_header_state);
-  s.setValue("sparkline_range", sparkline_range);
+  op(s, "chart_series_type", settings.chart_series_type);
-  s.setValue("multiple_lines_bytes", multiple_lines_bytes);
+  op(s, "theme", settings.theme);
-  s.setValue("log_livestream", log_livestream);
+  op(s, "sparkline_range", settings.sparkline_range);
-  s.setValue("log_path", log_path);
+  op(s, "multiple_lines_hex", settings.multiple_lines_hex);
-  s.setValue("drag_direction", drag_direction);
+  op(s, "log_livestream", settings.log_livestream);
-  s.setValue("suppress_defined_signals", suppress_defined_signals);
+  op(s, "log_path", settings.log_path);
-  s.sync();
+  op(s, "drag_direction", (int &)settings.drag_direction);
-  return s.status();
+  op(s, "suppress_defined_signals", settings.suppress_defined_signals);
 }
 
-void Settings::load() {
+Settings::Settings() {
-  QSettings s(filePath(), QSettings::IniFormat);
+  last_dir = last_route_dir = QDir::homePath();
-  fps = s.value("fps", 10).toInt();
+  log_path = QStandardPaths::writableLocation(QStandardPaths::HomeLocation) + "/cabana_live_stream/";
-  max_cached_minutes = s.value("max_cached_minutes", 30).toInt();
+  settings_op([](QSettings &s, const QString &key, auto &value) {
-  chart_height = s.value("chart_height", 200).toInt();
+    if (auto v = s.value(key); v.canConvert<std::decay_t<decltype(value)>>())
-  chart_range = s.value("chart_range", 3 * 60).toInt();
+      value = v.value<std::decay_t<decltype(value)>>();
-  chart_column_count = s.value("chart_column_count", 1).toInt();
+  });
-  last_dir = s.value("last_dir", QDir::homePath()).toString();
+}
-  last_route_dir = s.value("last_route_dir", QDir::homePath()).toString();
+
-  window_state = s.value("window_state").toByteArray();
+Settings::~Settings() {
-  geometry = s.value("geometry").toByteArray();
+  settings_op([](QSettings &s, const QString &key, auto &v) { s.setValue(key, v); });
-  video_splitter_state = s.value("video_splitter_state").toByteArray();
-  recent_files = s.value("recent_files").toStringList();
-  message_header_state = s.value("message_header_state_v3").toByteArray();
-  chart_series_type = s.value("chart_series_type", 0).toInt();
-  theme = s.value("theme", 0).toInt();
-  sparkline_range = s.value("sparkline_range", 15).toInt();
-  multiple_lines_bytes = s.value("multiple_lines_bytes", true).toBool();
-  log_livestream = s.value("log_livestream", true).toBool();
-  log_path = s.value("log_path").toString();
-  drag_direction = (Settings::DragDirection)s.value("drag_direction", 0).toInt();
-  suppress_defined_signals = s.value("suppress_defined_signals", false).toBool();
-  if (log_path.isEmpty()) {
-    log_path = QStandardPaths::writableLocation(QStandardPaths::HomeLocation) + "/cabana_live_stream/";
-  }
 }
 
 // SettingsDlg
@@ -73,45 +61,39 @@ SettingsDlg::SettingsDlg(QWidget *parent) : QDialog(parent) {
   QGroupBox *groupbox = new QGroupBox("General");
   QFormLayout *form_layout = new QFormLayout(groupbox);
 
-  theme = new QComboBox(this);
+  form_layout->addRow(tr("Color Theme"), theme = new QComboBox(this));
   theme->setToolTip(tr("You may need to restart cabana after changes theme"));
   theme->addItems({tr("Automatic"), tr("Light"), tr("Dark")});
   theme->setCurrentIndex(settings.theme);
-  form_layout->addRow(tr("Color Theme"), theme);
 
-  fps = new QSpinBox(this);
+  form_layout->addRow("FPS", fps = new QSpinBox(this));
   fps->setRange(10, 100);
   fps->setSingleStep(10);
   fps->setValue(settings.fps);
-  form_layout->addRow("FPS", fps);
 
-  cached_minutes = new QSpinBox(this);
+  form_layout->addRow(tr("Max Cached Minutes"), cached_minutes = new QSpinBox(this));
   cached_minutes->setRange(5, 60);
   cached_minutes->setSingleStep(1);
   cached_minutes->setValue(settings.max_cached_minutes);
-  form_layout->addRow(tr("Max Cached Minutes"), cached_minutes);
   main_layout->addWidget(groupbox);
 
   groupbox = new QGroupBox("New Signal Settings");
   form_layout = new QFormLayout(groupbox);
-  drag_direction = new QComboBox(this);
+  form_layout->addRow(tr("Drag Direction"), drag_direction = new QComboBox(this));
   drag_direction->addItems({tr("MSB First"), tr("LSB First"), tr("Always Little Endian"), tr("Always Big Endian")});
   drag_direction->setCurrentIndex(settings.drag_direction);
-  form_layout->addRow(tr("Drag Direction"), drag_direction);
   main_layout->addWidget(groupbox);
 
   groupbox = new QGroupBox("Chart");
   form_layout = new QFormLayout(groupbox);
-  chart_series_type = new QComboBox(this);
+  form_layout->addRow(tr("Default Series Type"), chart_series_type = new QComboBox(this));
   chart_series_type->addItems({tr("Line"), tr("Step Line"), tr("Scatter")});
   chart_series_type->setCurrentIndex(settings.chart_series_type);
-  form_layout->addRow(tr("Chart Default Series Type"), chart_series_type);
 
-  chart_height = new QSpinBox(this);
+  form_layout->addRow(tr("Chart Height"), chart_height = new QSpinBox(this));
   chart_height->setRange(100, 500);
   chart_height->setSingleStep(10);
   chart_height->setValue(settings.chart_height);
-  form_layout->addRow(tr("Chart Height"), chart_height);
   main_layout->addWidget(groupbox);
 
   log_livestream = new QGroupBox(tr("Enable live stream logging"), this);
@@ -123,10 +105,9 @@ SettingsDlg::SettingsDlg(QWidget *parent) : QDialog(parent) {
   path_layout->addWidget(browse_btn);
   main_layout->addWidget(log_livestream);
 
-
+  auto buttonBox = new QDialogButtonBox(QDialogButtonBox::Ok | QDialogButtonBox::Cancel);
-  auto buttonBox = new QDialogButtonBox(QDialogButtonBox::Ok | QDialogButtonBox::Cancel | QDialogButtonBox::Apply);
   main_layout->addWidget(buttonBox);
-  main_layout->addStretch(1);
+  setFixedSize(400, sizeHint().height());
 
   QObject::connect(browse_btn, &QPushButton::clicked, [this]() {
     QString fn = QFileDialog::getExistingDirectory(
@@ -137,31 +118,22 @@ SettingsDlg::SettingsDlg(QWidget *parent) : QDialog(parent) {
       log_path->setText(fn);
     }
   });
-  QObject::connect(buttonBox, &QDialogButtonBox::clicked, [=](QAbstractButton *button) {
+  QObject::connect(buttonBox, &QDialogButtonBox::rejected, this, &QDialog::reject);
-    auto role = buttonBox->buttonRole(button);
+  QObject::connect(buttonBox, &QDialogButtonBox::accepted, this, &SettingsDlg::save);
-    if (role == QDialogButtonBox::AcceptRole) {
-      save();
-      accept();
-    } else if (role == QDialogButtonBox::ApplyRole) {
-      save();
-    } else if (role == QDialogButtonBox::RejectRole) {
-      reject();
-    }
-  });
 }
 
 void SettingsDlg::save() {
-  settings.fps = fps->value();
   if (std::exchange(settings.theme, theme->currentIndex()) != settings.theme) {
     // set theme before emit changed
     utils::setTheme(settings.theme);
   }
+  settings.fps = fps->value();
   settings.max_cached_minutes = cached_minutes->value();
   settings.chart_series_type = chart_series_type->currentIndex();
   settings.chart_height = chart_height->value();
   settings.log_livestream = log_livestream->isChecked();
   settings.log_path = log_path->text();
   settings.drag_direction = (Settings::DragDirection)drag_direction->currentIndex();
-  settings.save();
   emit settings.changed();
+  QDialog::accept();
 }

tools/cabana/settings.h

@@ -1,13 +1,10 @@
 #pragma once
 
-#include <QApplication>
 #include <QByteArray>
-#include <QCheckBox>
 #include <QComboBox>
 #include <QDialog>
 #include <QGroupBox>
 #include <QLineEdit>
-#include <QSettings>
 #include <QSpinBox>
 
 #define LIGHT_THEME 1
@@ -24,11 +21,10 @@ public:
     AlwaysBE,
   };
 
-  Settings() {}
+  Settings();
-  QSettings::Status save();
+  ~Settings();
-  void load();
-  inline static QString filePath() { return QApplication::applicationDirPath() + "/settings"; }
 
+  bool absolute_time = false;
   int fps = 10;
   int max_cached_minutes = 30;
   int chart_height = 200;
@@ -37,7 +33,7 @@ public:
   int chart_series_type = 0;
   int theme = 0;
   int sparkline_range = 15; // 15 seconds
-  bool multiple_lines_bytes = true;
+  bool multiple_lines_hex = false;
   bool log_livestream = true;
   bool suppress_defined_signals = false;
   QString log_path;
@@ -48,15 +44,13 @@ public:
   QByteArray window_state;
   QStringList recent_files;
   QByteArray message_header_state;
-  DragDirection drag_direction;
+  DragDirection drag_direction = MsbFirst;
 
 signals:
   void changed();
 };
 
 class SettingsDlg : public QDialog {
-  Q_OBJECT
-
 public:
   SettingsDlg(QWidget *parent);
   void save();

tools/cabana/signalview.cc

@@ -36,8 +36,8 @@ SignalModel::SignalModel(QObject *parent) : root(new Item), QAbstractItemModel(p
 void SignalModel::insertItem(SignalModel::Item *parent_item, int pos, const cabana::Signal *sig) {
   Item *item = new Item{.sig = sig, .parent = parent_item, .title = sig->name, .type = Item::Sig};
   parent_item->children.insert(pos, item);
-  QString titles[]{"Name", "Size", "Node", "Little Endian", "Signed", "Offset", "Factor", "Type", "Multiplex Value", "Extra Info",
+  QString titles[]{"Name", "Size", "Receiver Nodes", "Little Endian", "Signed", "Offset", "Factor", "Type",
-                   "Unit", "Comment", "Minimum Value", "Maximum Value", "Value Descriptions"};
+                   "Multiplex Value", "Extra Info", "Unit", "Comment", "Minimum Value", "Maximum Value", "Value Table"};
   for (int i = 0; i < std::size(titles); ++i) {
     item->children.push_back(new Item{.sig = sig, .parent = item, .title = titles[i], .type = (Item::Type)(i + Item::Name)});
   }
@@ -68,10 +68,7 @@ void SignalModel::refresh() {
 }
 
 SignalModel::Item *SignalModel::getItem(const QModelIndex &index) const {
-  SignalModel::Item *item = nullptr;
+  auto item = index.isValid() ? (SignalModel::Item *)index.internalPointer() : nullptr;
-  if (index.isValid()) {
-    item = (SignalModel::Item *)index.internalPointer();
-  }
   return item ? item : root.get();
 }
 
@@ -369,8 +366,7 @@ void SignalItemDelegate::paint(QPainter *painter, const QStyleOptionViewItem &op
         painter->setFont(label_font);
         QString freq = QString("%1 hz").arg(item->sparkline.freq(), 0, 'g', 2);
         painter->drawText(rect.adjusted(5, 0, 0, 0), Qt::AlignLeft | Qt::AlignVCenter, freq);
-        QFontMetrics fm(label_font);
+        value_adjust = QFontMetrics(label_font).width(freq) + 10;
-        value_adjust = fm.width(freq) + 10;
       }
       // signal value
       painter->setFont(option.font);
@@ -447,7 +443,7 @@ SignalView::SignalView(ChartsWidget *charts, QWidget *parent) : charts(charts),
   QRegularExpression re("\\S+");
   filter_edit->setValidator(new QRegularExpressionValidator(re, this));
   filter_edit->setClearButtonEnabled(true);
-  filter_edit->setPlaceholderText(tr("filter signals"));
+  filter_edit->setPlaceholderText(tr("Filter Signal"));
   hl->addWidget(filter_edit);
   hl->addStretch(1);
 
@@ -502,6 +498,12 @@ SignalView::SignalView(ChartsWidget *charts, QWidget *parent) : charts(charts),
   QObject::connect(tree->verticalScrollBar(), &QScrollBar::valueChanged, [this]() { updateState(); });
   QObject::connect(tree->verticalScrollBar(), &QScrollBar::rangeChanged, [this]() { updateState(); });
   QObject::connect(can, &AbstractStream::msgsReceived, this, &SignalView::updateState);
+  QObject::connect(tree->header(), &QHeaderView::sectionResized, [this](int logicalIndex, int oldSize, int newSize) {
+    if (logicalIndex == 1) {
+      value_column_width = newSize;
+      updateState();
+    }
+  });
 
   setWhatsThis(tr(R"(
     <b>Signal view</b><br />
@@ -616,13 +618,13 @@ void SignalView::handleSignalUpdated(const cabana::Signal *sig) {
   }
 }
 
-void SignalView::updateState(const QHash<MessageId, CanData> *msgs) {
+void SignalView::updateState(const std::set<MessageId> *msgs) {
   const auto &last_msg = can->lastMessage(model->msg_id);
-  if (model->rowCount() == 0 || (msgs && !msgs->contains(model->msg_id)) || last_msg.dat.size() == 0) return;
+  if (model->rowCount() == 0 || (msgs && !msgs->count(model->msg_id)) || last_msg.dat.size() == 0) return;
 
   for (auto item : model->root->children) {
     double value = 0;
-    if (item->sig->getValue((uint8_t *)last_msg.dat.constData(), last_msg.dat.size(), &value)) {
+    if (item->sig->getValue(last_msg.dat.data(), last_msg.dat.size(), &value)) {
       item->sig_val = item->sig->formatValue(value);
     }
     max_value_width = std::max(max_value_width, fontMetrics().width(item->sig_val));
@@ -638,11 +640,11 @@ void SignalView::updateState(const QHash<MessageId, CanData> *msgs) {
       last_visible_row = bottom.parent().isValid() ? bottom.parent().row() : bottom.row();
     }
 
-    QSize size(tree->columnWidth(1) - delegate->button_size.width(), delegate->button_size.height());
+    const static int min_max_width = QFontMetrics(delegate->minmax_font).width("-000.00") + 5;
-    int min_max_width = std::min(size.width() - 10, QFontMetrics(delegate->minmax_font).width("-000.00") + 5);
+    int available_width = value_column_width - delegate->button_size.width();
-    int value_width = std::min<int>(max_value_width, size.width() * 0.35);
+    int value_width = std::min<int>(max_value_width + min_max_width, available_width / 2);
-    size -= {value_width + min_max_width, style()->pixelMetric(QStyle::PM_FocusFrameVMargin) * 2};
+    QSize size(available_width - value_width,
-
+               delegate->button_size.height() - style()->pixelMetric(QStyle::PM_FocusFrameVMargin) * 2);
     QFutureSynchronizer<void> synchronizer;
     for (int i = first_visible_row; i <= last_visible_row; ++i) {
       auto item = model->getItem(model->index(i, 1));

tools/cabana/signalview.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <memory>
+#include <set>
 
 #include <QAbstractItemModel>
 #include <QLabel>
@@ -82,7 +83,7 @@ class SignalItemDelegate : public QStyledItemDelegate {
 public:
   SignalItemDelegate(QObject *parent);
   void paint(QPainter *painter, const QStyleOptionViewItem &option, const QModelIndex &index) const override;
-  QSize sizeHint(const QStyleOptionViewItem &option, const QModelIndex &index) const;
+  QSize sizeHint(const QStyleOptionViewItem &option, const QModelIndex &index) const override;
   QWidget *createEditor(QWidget *parent, const QStyleOptionViewItem &option, const QModelIndex &index) const override;
   void updateEditorGeometry(QWidget *editor, const QStyleOptionViewItem &option, const QModelIndex &index) const override;
   void setModelData(QWidget *editor, QAbstractItemModel *model, const QModelIndex &index) const override;
@@ -117,7 +118,7 @@ private:
   void setSparklineRange(int value);
   void handleSignalAdded(MessageId id, const cabana::Signal *sig);
   void handleSignalUpdated(const cabana::Signal *sig);
-  void updateState(const QHash<MessageId, CanData> *msgs = nullptr);
+  void updateState(const std::set<MessageId> *msgs = nullptr);
 
   struct TreeView : public QTreeView {
     TreeView(QWidget *parent) : QTreeView(parent) {}
@@ -136,6 +137,7 @@ private:
     }
   };
   int max_value_width = 0;
+  int value_column_width = 0;
   TreeView *tree;
   QLabel *sparkline_label;
   QSlider *sparkline_range_slider;

tools/cabana/streams/abstractstream.cc

@@ -1,9 +1,10 @@
 #include "tools/cabana/streams/abstractstream.h"
 
 #include <algorithm>
-#include <vector>
+#include <utility>
 
-#include <QTimer>
+#include "common/timing.h"
+#include "tools/cabana/settings.h"
 
 static const int EVENT_NEXT_BUFFER_SIZE = 6 * 1024 * 1024;  // 6MB
 
@@ -16,82 +17,97 @@ StreamNotifier *StreamNotifier::instance() {
 
 AbstractStream::AbstractStream(QObject *parent) : QObject(parent) {
   assert(parent != nullptr);
-  new_msgs = std::make_unique<QHash<MessageId, CanData>>();
+  event_buffer_ = std::make_unique<MonotonicBuffer>(EVENT_NEXT_BUFFER_SIZE);
-  event_buffer = std::make_unique<MonotonicBuffer>(EVENT_NEXT_BUFFER_SIZE);
 
+  QObject::connect(this, &AbstractStream::privateUpdateLastMsgsSignal, this, &AbstractStream::updateLastMessages, Qt::QueuedConnection);
   QObject::connect(this, &AbstractStream::seekedTo, this, &AbstractStream::updateLastMsgsTo);
-  QObject::connect(&settings, &Settings::changed, this, &AbstractStream::updateMasks);
   QObject::connect(dbc(), &DBCManager::DBCFileChanged, this, &AbstractStream::updateMasks);
   QObject::connect(dbc(), &DBCManager::maskUpdated, this, &AbstractStream::updateMasks);
   QObject::connect(this, &AbstractStream::streamStarted, [this]() {
     emit StreamNotifier::instance()->changingStream();
     delete can;
     can = this;
-    // TODO: add method stop() to class AbstractStream
-    QObject::connect(qApp, &QApplication::aboutToQuit, can, []() {
-      qDebug() << "stopping stream thread";
-      can->pause(true);
-    });
     emit StreamNotifier::instance()->streamStarted();
   });
 }
 
 void AbstractStream::updateMasks() {
-  std::lock_guard lk(mutex);
+  std::lock_guard lk(mutex_);
-  masks.clear();
+  masks_.clear();
-  if (settings.suppress_defined_signals) {
+  if (!settings.suppress_defined_signals)
-    for (auto s : sources) {
+    return;
-      if (auto f = dbc()->findDBCFile(s)) {
+
-        for (const auto &[address, m] : f->getMessages()) {
+  for (const auto s : sources) {
-          masks[{.source = (uint8_t)s, .address = address}] = m.mask;
+    for (const auto &[address, m] : dbc()->getMessages(s)) {
-        }
+      masks_[{.source = (uint8_t)s, .address = address}] = m.mask;
+    }
+  }
+  // clear bit change counts
+  for (auto &[id, m] : messages_) {
+    auto &mask = masks_[id];
+    const int size = std::min(mask.size(), m.last_changes.size());
+    for (int i = 0; i < size; ++i) {
+      for (int j = 0; j < 8; ++j) {
+        if (((mask[i] >> (7 - j)) & 1) != 0) m.last_changes[i].bit_change_counts[j] = 0;
       }
     }
   }
 }
 
-void AbstractStream::updateMessages(QHash<MessageId, CanData> *messages) {
+void AbstractStream::suppressDefinedSignals(bool suppress) {
+  settings.suppress_defined_signals = suppress;
+  updateMasks();
+}
+
+size_t AbstractStream::suppressHighlighted() {
+  std::lock_guard lk(mutex_);
+  size_t cnt = 0;
+  const double cur_ts = currentSec();
+  for (auto &[_, m] : messages_) {
+    for (auto &last_change : m.last_changes) {
+      const double dt = cur_ts - last_change.ts;
+      if (dt < 2.0) {
+        last_change.suppressed = true;
+      }
+      // clear bit change counts
+      last_change.bit_change_counts.fill(0);
+      cnt += last_change.suppressed;
+    }
+  }
+  return cnt;
+}
+
+void AbstractStream::clearSuppressed() {
+  std::lock_guard lk(mutex_);
+  for (auto &[_, m] : messages_) {
+    std::for_each(m.last_changes.begin(), m.last_changes.end(), [](auto &c) { c.suppressed = false; });
+  }
+}
+
+void AbstractStream::updateLastMessages() {
   auto prev_src_size = sources.size();
   auto prev_msg_size = last_msgs.size();
-  for (auto it = messages->begin(); it != messages->end(); ++it) {
+  std::set<MessageId> msgs;
-    const auto &id = it.key();
+  {
-    last_msgs[id] = it.value();
+    std::lock_guard lk(mutex_);
-    sources.insert(id.source);
+    for (const auto &id : new_msgs_) {
+      last_msgs[id] = messages_[id];
+      sources.insert(id.source);
+    }
+    msgs = std::move(new_msgs_);
   }
+
   if (sources.size() != prev_src_size) {
     updateMasks();
     emit sourcesUpdated(sources);
   }
-  emit updated();
+  emit msgsReceived(&msgs, prev_msg_size != last_msgs.size());
-  emit msgsReceived(messages, prev_msg_size != last_msgs.size());
-  delete messages;
-  processing = false;
 }
 
 void AbstractStream::updateEvent(const MessageId &id, double sec, const uint8_t *data, uint8_t size) {
-  std::lock_guard lk(mutex);
+  std::lock_guard lk(mutex_);
-  auto mask_it = masks.find(id);
+  messages_[id].compute(id, data, size, sec, getSpeed(), masks_[id]);
-  std::vector<uint8_t> *mask = mask_it == masks.end() ? nullptr : &mask_it->second;
+  new_msgs_.insert(id);
-  all_msgs[id].compute(id, (const char *)data, size, sec, getSpeed(), mask);
-  if (!new_msgs->contains(id)) {
-    new_msgs->insert(id, {});
-  }
-}
-
-bool AbstractStream::postEvents() {
-  // delay posting CAN message if UI thread is busy
-  if (processing == false) {
-    processing = true;
-    for (auto it = new_msgs->begin(); it != new_msgs->end(); ++it) {
-      it.value() = all_msgs[it.key()];
-    }
-    // use pointer to avoid data copy in queued connection.
-    QMetaObject::invokeMethod(this, std::bind(&AbstractStream::updateMessages, this, new_msgs.release()), Qt::QueuedConnection);
-    new_msgs.reset(new QHash<MessageId, CanData>);
-    new_msgs->reserve(100);
-    return true;
-  }
-  return false;
 }
 
 const std::vector<const CanEvent *> &AbstractStream::events(const MessageId &id) const {
@@ -103,93 +119,81 @@ const std::vector<const CanEvent *> &AbstractStream::events(const MessageId &id)
 const CanData &AbstractStream::lastMessage(const MessageId &id) {
   static CanData empty_data = {};
   auto it = last_msgs.find(id);
-  return it != last_msgs.end() ? it.value() : empty_data;
+  return it != last_msgs.end() ? it->second : empty_data;
 }
 
 // it is thread safe to update data in updateLastMsgsTo.
 // updateLastMsgsTo is always called in UI thread.
 void AbstractStream::updateLastMsgsTo(double sec) {
-  new_msgs.reset(new QHash<MessageId, CanData>);
+  new_msgs_.clear();
-  all_msgs.clear();
+  messages_.clear();
-  last_msgs.clear();
 
   uint64_t last_ts = (sec + routeStartTime()) * 1e9;
-  for (auto &[id, ev] : events_) {
+  for (const auto &[id, ev] : events_) {
-    auto it = std::lower_bound(ev.crbegin(), ev.crend(), last_ts, [](auto e, uint64_t ts) {
+    auto it = std::upper_bound(ev.begin(), ev.end(), last_ts, CompareCanEvent());
-      return e->mono_time > ts;
+    if (it != ev.begin()) {
-    });
+      auto prev = std::prev(it);
-    auto mask_it = masks.find(id);
+      double ts = (*prev)->mono_time / 1e9 - routeStartTime();
-    std::vector<uint8_t> *mask = mask_it == masks.end() ? nullptr : &mask_it->second;
+      auto &m = messages_[id];
-    if (it != ev.crend()) {
+      m.compute(id, (*prev)->dat, (*prev)->size, ts, getSpeed(), {});
-      double ts = (*it)->mono_time / 1e9 - routeStartTime();
+      m.count = std::distance(ev.begin(), prev) + 1;
-      auto &m = all_msgs[id];
-      m.compute(id, (const char *)(*it)->dat, (*it)->size, ts, getSpeed(), mask);
-      m.count = std::distance(it, ev.crend());
     }
   }
 
-  // deep copy all_msgs to last_msgs to avoid multi-threading issue.
+  bool id_changed = messages_.size() != last_msgs.size() ||
-  last_msgs = all_msgs;
+                    std::any_of(messages_.cbegin(), messages_.cend(),
-  last_msgs.detach();
+                                [this](const auto &m) { return !last_msgs.count(m.first); });
-  // use a timer to prevent recursive calls
+  last_msgs = messages_;
-  QTimer::singleShot(0, [this]() {
+  emit msgsReceived(nullptr, id_changed);
-    emit updated();
-    emit msgsReceived(&last_msgs, true);
-  });
 }
 
-void AbstractStream::mergeEvents(std::vector<Event *>::const_iterator first, std::vector<Event *>::const_iterator last) {
+const CanEvent *AbstractStream::newEvent(uint64_t mono_time, const cereal::CanData::Reader &c) {
-  static std::unordered_map<MessageId, std::deque<const CanEvent *>> new_events_map;
+  auto dat = c.getDat();
-  static  std::vector<const CanEvent *> new_events;
+  CanEvent *e = (CanEvent *)event_buffer_->allocate(sizeof(CanEvent) + sizeof(uint8_t) * dat.size());
-  new_events_map.clear();
+  e->src = c.getSrc();
-  new_events.clear();
+  e->address = c.getAddress();
-
+  e->mono_time = mono_time;
-  for (auto it = first; it != last; ++it) {
+  e->size = dat.size();
-    if ((*it)->which == cereal::Event::Which::CAN) {
+  memcpy(e->dat, (uint8_t *)dat.begin(), e->size);
-      uint64_t ts = (*it)->mono_time;
+  return e;
-      for (const auto &c : (*it)->event.getCan()) {
+}
-        auto dat = c.getDat();
-        CanEvent *e = (CanEvent *)event_buffer->allocate(sizeof(CanEvent) + sizeof(uint8_t) * dat.size());
-        e->src = c.getSrc();
-        e->address = c.getAddress();
-        e->mono_time = ts;
-        e->size = dat.size();
-        memcpy(e->dat, (uint8_t *)dat.begin(), e->size);
-
-        new_events_map[{.source = e->src, .address = e->address}].push_back(e);
-        new_events.push_back(e);
-      }
-    }
-  }
 
-  for (auto &[id, new_e] : new_events_map) {
+void AbstractStream::mergeEvents(const std::vector<const CanEvent *> &events) {
-    auto &e = events_[id];
+  static MessageEventsMap msg_events;
-    auto insert_pos = std::upper_bound(e.cbegin(), e.cend(), new_e.front()->mono_time, CompareCanEvent());
+  std::for_each(msg_events.begin(), msg_events.end(), [](auto &e) { e.second.clear(); });
-    e.insert(insert_pos, new_e.cbegin(), new_e.cend());
+  for (auto e : events) {
+    msg_events[{.source = e->src, .address = e->address}].push_back(e);
   }
 
-  if (!new_events.empty()) {
+  if (!events.empty()) {
-    auto insert_pos = std::upper_bound(all_events_.cbegin(), all_events_.cend(), new_events.front()->mono_time, CompareCanEvent());
+    for (const auto &[id, new_e] : msg_events) {
-    all_events_.insert(insert_pos, new_events.cbegin(), new_events.cend());
+      if (!new_e.empty()) {
+        auto &e = events_[id];
+        auto pos = std::upper_bound(e.cbegin(), e.cend(), new_e.front()->mono_time, CompareCanEvent());
+        e.insert(pos, new_e.cbegin(), new_e.cend());
+      }
+    }
+    auto pos = std::upper_bound(all_events_.cbegin(), all_events_.cend(), events.front()->mono_time, CompareCanEvent());
+    all_events_.insert(pos, events.cbegin(), events.cend());
+    emit eventsMerged(msg_events);
   }
-
   lastest_event_ts = all_events_.empty() ? 0 : all_events_.back()->mono_time;
-  emit eventsMerged();
 }
 
 // CanData
 
 namespace {
 
-constexpr int periodic_threshold = 10;
+enum Color { GREYISH_BLUE, CYAN, RED};
-constexpr int start_alpha = 128;
+QColor getColor(int c) {
-constexpr float fade_time = 2.0;
+  constexpr int start_alpha = 128;
-const QColor CYAN = QColor(0, 187, 255, start_alpha);
+  static const QColor colors[] = {
-const QColor RED = QColor(255, 0, 0, start_alpha);
+      [GREYISH_BLUE] = QColor(102, 86, 169, start_alpha / 2),
-const QColor GREYISH_BLUE = QColor(102, 86, 169, start_alpha / 2);
+      [CYAN] = QColor(0, 187, 255, start_alpha),
-const QColor CYAN_LIGHTER = QColor(0, 187, 255, start_alpha).lighter(135);
+      [RED] = QColor(255, 0, 0, start_alpha),
-const QColor RED_LIGHTER = QColor(255, 0, 0, start_alpha).lighter(135);
+  };
-const QColor GREYISH_BLUE_LIGHTER = QColor(102, 86, 169, start_alpha / 2).lighter(135);
+  return settings.theme == LIGHT_THEME ? colors[c] : colors[c].lighter(135);
+}
 
 inline QColor blend(const QColor &a, const QColor &b) {
   return QColor((a.red() + b.red()) / 2, (a.green() + b.green()) / 2, (a.blue() + b.blue()) / 2, (a.alpha() + b.alpha()) / 2);
@@ -212,8 +216,8 @@ double calc_freq(const MessageId &msg_id, double current_sec) {
 
 }  // namespace
 
-void CanData::compute(const MessageId &msg_id, const char *can_data, const int size, double current_sec,
+void CanData::compute(const MessageId &msg_id, const uint8_t *can_data, const int size, double current_sec,
-                      double playback_speed, const std::vector<uint8_t> *mask, double in_freq) {
+                      double playback_speed, const std::vector<uint8_t> &mask, double in_freq) {
   ts = current_sec;
   ++count;
 
@@ -224,55 +228,53 @@ void CanData::compute(const MessageId &msg_id, const char *can_data, const int s
 
   if (dat.size() != size) {
     dat.resize(size);
-    bit_change_counts.resize(size);
+    colors.assign(size, QColor(0, 0, 0, 0));
-    colors = QVector(size, QColor(0, 0, 0, 0));
+    last_changes.resize(size);
-    last_change_t.assign(size, ts);
+    std::for_each(last_changes.begin(), last_changes.end(), [current_sec](auto &c) { c.ts = current_sec; });
-    last_delta.resize(size);
-    same_delta_counter.resize(size);
   } else {
-    bool lighter = settings.theme == DARK_THEME;
+    constexpr int periodic_threshold = 10;
-    const QColor &cyan = !lighter ? CYAN : CYAN_LIGHTER;
+    constexpr float fade_time = 2.0;
-    const QColor &red = !lighter ? RED : RED_LIGHTER;
+    const float alpha_delta = 1.0 / (freq + 1) / (fade_time * playback_speed);
-    const QColor &greyish_blue = !lighter ? GREYISH_BLUE : GREYISH_BLUE_LIGHTER;
 
     for (int i = 0; i < size; ++i) {
-      const uint8_t mask_byte = (mask && i < mask->size()) ? (~((*mask)[i])) : 0xff;
+      auto &last_change = last_changes[i];
+
+      uint8_t mask_byte = last_change.suppressed ? 0x00 : 0xFF;
+      if (i < mask.size()) mask_byte &= ~(mask[i]);
+
       const uint8_t last = dat[i] & mask_byte;
       const uint8_t cur = can_data[i] & mask_byte;
-      const int delta = cur - last;
-
       if (last != cur) {
-        double delta_t = ts - last_change_t[i];
+        const int delta = cur - last;
-
         // Keep track if signal is changing randomly, or mostly moving in the same direction
-        if (std::signbit(delta) == std::signbit(last_delta[i])) {
+        if (std::signbit(delta) == std::signbit(last_change.delta)) {
-          same_delta_counter[i] = std::min(16, same_delta_counter[i] + 1);
+          last_change.same_delta_counter = std::min(16, last_change.same_delta_counter + 1);
         } else {
-          same_delta_counter[i] = std::max(0, same_delta_counter[i] - 4);
+          last_change.same_delta_counter = std::max(0, last_change.same_delta_counter - 4);
         }
 
+        const double delta_t = ts - last_change.ts;
         // Mostly moves in the same direction, color based on delta up/down
-        if (delta_t * freq > periodic_threshold || same_delta_counter[i] > 8) {
+        if (delta_t * freq > periodic_threshold || last_change.same_delta_counter > 8) {
           // Last change was while ago, choose color based on delta up or down
-          colors[i] = (cur > last) ? cyan : red;
+          colors[i] = getColor(cur > last ? CYAN : RED);
         } else {
           // Periodic changes
-          colors[i] = blend(colors[i], greyish_blue);
+          colors[i] = blend(colors[i], getColor(GREYISH_BLUE));
         }
 
         // Track bit level changes
         const uint8_t tmp = (cur ^ last);
         for (int bit = 0; bit < 8; bit++) {
-          if (tmp & (1 << bit)) {
+          if (tmp & (1 << (7 - bit))) {
-            bit_change_counts[i][bit] += 1;
+            last_change.bit_change_counts[bit] += 1;
           }
         }
 
-        last_change_t[i] = ts;
+        last_change.ts = ts;
-        last_delta[i] = delta;
+        last_change.delta = delta;
       } else {
         // Fade out
-        float alpha_delta = 1.0 / (freq + 1) / (fade_time * playback_speed);
         colors[i].setAlphaF(std::max(0.0, colors[i].alphaF() - alpha_delta));
       }
     }

tools/cabana/streams/abstractstream.h

@@ -1,35 +1,37 @@
 #pragma once
 
 #include <array>
-#include <atomic>
-#include <deque>
 #include <memory>
-#include <tuple>
+#include <mutex>
+#include <set>
 #include <unordered_map>
 #include <vector>
 
 #include <QColor>
-#include <QHash>
+#include <QDateTime>
 
-#include "common/timing.h"
+#include "cereal/messaging/messaging.h"
 #include "tools/cabana/dbc/dbcmanager.h"
-#include "tools/cabana/settings.h"
 #include "tools/cabana/util.h"
-#include "tools/replay/replay.h"
 
 struct CanData {
-  void compute(const MessageId &msg_id, const char *dat, const int size, double current_sec,
+  void compute(const MessageId &msg_id, const uint8_t *dat, const int size, double current_sec,
-               double playback_speed, const std::vector<uint8_t> *mask = nullptr, double in_freq = 0);
+               double playback_speed, const std::vector<uint8_t> &mask, double in_freq = 0);
 
   double ts = 0.;
   uint32_t count = 0;
   double freq = 0;
-  QByteArray dat;
+  std::vector<uint8_t> dat;
-  QVector<QColor> colors;
+  std::vector<QColor> colors;
-  std::vector<double> last_change_t;
+
-  std::vector<std::array<uint32_t, 8>> bit_change_counts;
+  struct ByteLastChange {
-  std::vector<int> last_delta;
+    double ts;
-  std::vector<int> same_delta_counter;
+    int delta;
+    int same_delta_counter;
+    bool suppressed;
+    std::array<uint32_t, 8> bit_change_counts;
+  };
+  std::vector<ByteLastChange> last_changes;
   double last_freq_update_ts = 0;
 };
 
@@ -52,6 +54,8 @@ struct BusConfig {
   bool can_fd = false;
 };
 
+typedef std::unordered_map<MessageId, std::vector<const CanEvent *>> MessageEventsMap;
+
 class AbstractStream : public QObject {
   Q_OBJECT
 
@@ -59,56 +63,65 @@ public:
   AbstractStream(QObject *parent);
   virtual ~AbstractStream() {}
   virtual void start() = 0;
-  inline bool liveStreaming() const { return route() == nullptr; }
+  virtual bool liveStreaming() const { return true; }
   virtual void seekTo(double ts) {}
   virtual QString routeName() const = 0;
   virtual QString carFingerprint() const { return ""; }
+  virtual QDateTime beginDateTime() const { return {}; }
   virtual double routeStartTime() const { return 0; }
   virtual double currentSec() const = 0;
   virtual double totalSeconds() const { return lastEventMonoTime() / 1e9 - routeStartTime(); }
-  const CanData &lastMessage(const MessageId &id);
-  virtual VisionStreamType visionStreamType() const { return VISION_STREAM_ROAD; }
-  virtual const Route *route() const { return nullptr; }
   virtual void setSpeed(float speed) {}
   virtual double getSpeed() { return 1; }
   virtual bool isPaused() const { return false; }
   virtual void pause(bool pause) {}
-  const std::vector<const CanEvent *> &allEvents() const { return all_events_; }
+
+  inline const std::unordered_map<MessageId, CanData> &lastMessages() const { return last_msgs; }
+  inline const MessageEventsMap &eventsMap() const { return events_; }
+  inline const std::vector<const CanEvent *> &allEvents() const { return all_events_; }
+  const CanData &lastMessage(const MessageId &id);
   const std::vector<const CanEvent *> &events(const MessageId &id) const;
-  virtual const std::vector<std::tuple<double, double, TimelineType>> getTimeline() { return {}; }
+
+  size_t suppressHighlighted();
+  void clearSuppressed();
+  void suppressDefinedSignals(bool suppress);
 
 signals:
   void paused();
   void resume();
   void seekedTo(double sec);
   void streamStarted();
-  void eventsMerged();
+  void eventsMerged(const MessageEventsMap &events_map);
-  void updated();
+  void msgsReceived(const std::set<MessageId> *new_msgs, bool has_new_ids);
-  void msgsReceived(const QHash<MessageId, CanData> *new_msgs, bool has_new_ids);
   void sourcesUpdated(const SourceSet &s);
+  void privateUpdateLastMsgsSignal();
 
 public:
-  QHash<MessageId, CanData> last_msgs;
   SourceSet sources;
 
 protected:
-  void mergeEvents(std::vector<Event *>::const_iterator first, std::vector<Event *>::const_iterator last);
+  void mergeEvents(const std::vector<const CanEvent *> &events);
-  bool postEvents();
+  const CanEvent *newEvent(uint64_t mono_time, const cereal::CanData::Reader &c);
-  uint64_t lastEventMonoTime() const { return lastest_event_ts; }
   void updateEvent(const MessageId &id, double sec, const uint8_t *data, uint8_t size);
-  void updateMessages(QHash<MessageId, CanData> *);
+  uint64_t lastEventMonoTime() const { return lastest_event_ts; }
-  void updateMasks();
-  void updateLastMsgsTo(double sec);
 
-  uint64_t lastest_event_ts = 0;
-  std::atomic<bool> processing = false;
-  std::unique_ptr<QHash<MessageId, CanData>> new_msgs;
-  QHash<MessageId, CanData> all_msgs;
-  std::unordered_map<MessageId, std::vector<const CanEvent *>> events_;
   std::vector<const CanEvent *> all_events_;
-  std::unique_ptr<MonotonicBuffer> event_buffer;
+  uint64_t lastest_event_ts = 0;
-  std::mutex mutex;
+
-  std::unordered_map<MessageId, std::vector<uint8_t>> masks;
+private:
+  void updateLastMessages();
+  void updateLastMsgsTo(double sec);
+  void updateMasks();
+
+  MessageEventsMap events_;
+  std::unordered_map<MessageId, CanData> last_msgs;
+  std::unique_ptr<MonotonicBuffer> event_buffer_;
+
+  // Members accessed in multiple threads. (mutex protected)
+  std::mutex mutex_;
+  std::set<MessageId> new_msgs_;
+  std::unordered_map<MessageId, CanData> messages_;
+  std::unordered_map<MessageId, std::vector<uint8_t>> masks_;
 };
 
 class AbstractOpenStreamWidget : public QWidget {

tools/cabana/streams/devicestream.cc

@@ -8,6 +8,7 @@
 #include <QRadioButton>
 #include <QRegularExpression>
 #include <QRegularExpressionValidator>
+#include <QThread>
 
 // DeviceStream
 
@@ -21,17 +22,14 @@ void DeviceStream::streamThread() {
   std::string address = zmq_address.isEmpty() ? "127.0.0.1" : zmq_address.toStdString();
   std::unique_ptr<SubSocket> sock(SubSocket::create(context.get(), "can", address));
   assert(sock != NULL);
-  sock->setTimeout(50);
   // run as fast as messages come in
   while (!QThread::currentThread()->isInterruptionRequested()) {
-    Message *msg = sock->receive(true);
+    std::unique_ptr<Message> msg(sock->receive(true));
     if (!msg) {
       QThread::msleep(50);
       continue;
     }
-
+    handleEvent(kj::ArrayPtr<capnp::word>((capnp::word*)msg->getData(), msg->getSize() / sizeof(capnp::word)));
-    handleEvent(msg->getData(), msg->getSize());
-    delete msg;
   }
 }
 

tools/cabana/streams/livestream.cc

@@ -1,12 +1,17 @@
 #include "tools/cabana/streams/livestream.h"
 
+#include <QThread>
 #include <algorithm>
+#include <fstream>
 #include <memory>
 
+#include "common/timing.h"
+#include "common/util.h"
+
 struct LiveStream::Logger {
   Logger() : start_ts(seconds_since_epoch()), segment_num(-1) {}
 
-  void write(const char *data, const size_t size) {
+  void write(kj::ArrayPtr<capnp::word> data) {
     int n = (seconds_since_epoch() - start_ts) / 60.0;
     if (std::exchange(segment_num, n) != segment_num) {
       QString dir = QString("%1/%2--%3")
@@ -17,7 +22,8 @@ struct LiveStream::Logger {
       fs.reset(new std::ofstream((dir + "/rlog").toStdString(), std::ios::binary | std::ios::out));
     }
 
-    fs->write(data, size);
+    auto bytes = data.asBytes();
+    fs->write((const char*)bytes.begin(), bytes.size());
   }
 
   std::unique_ptr<std::ofstream> fs;
@@ -46,6 +52,7 @@ void LiveStream::start() {
   emit streamStarted();
   stream_thread->start();
   startUpdateTimer();
+  begin_date_time = QDateTime::currentDateTime();
 }
 
 LiveStream::~LiveStream() {
@@ -56,14 +63,20 @@ LiveStream::~LiveStream() {
 }
 
 // called in streamThread
-void LiveStream::handleEvent(const char *data, const size_t size) {
+void LiveStream::handleEvent(kj::ArrayPtr<capnp::word> data) {
   if (logger) {
-    logger->write(data, size);
+    logger->write(data);
   }
 
-  std::lock_guard lk(lock);
+  capnp::FlatArrayMessageReader reader(data);
-  auto &msg = receivedMessages.emplace_back(data, size);
+  auto event = reader.getRoot<cereal::Event>();
-  receivedEvents.push_back(msg.event);
+  if (event.which() == cereal::Event::Which::CAN) {
+    const uint64_t mono_time = event.getLogMonoTime();
+    std::lock_guard lk(lock);
+    for (const auto &c : event.getCan()) {
+      received_events_.push_back(newEvent(mono_time, c));
+    }
+  }
 }
 
 void LiveStream::timerEvent(QTimerEvent *event) {
@@ -71,9 +84,8 @@ void LiveStream::timerEvent(QTimerEvent *event) {
     {
       // merge events received from live stream thread.
       std::lock_guard lk(lock);
-      mergeEvents(receivedEvents.cbegin(), receivedEvents.cend());
+      mergeEvents(received_events_);
-      receivedEvents.clear();
+      received_events_.clear();
-      receivedMessages.clear();
     }
     if (!all_events_.empty()) {
       begin_event_ts = all_events_.front()->mono_time;
@@ -111,7 +123,7 @@ void LiveStream::updateEvents() {
     updateEvent(id, (e->mono_time - begin_event_ts) / 1e9, e->dat, e->size);
     current_event_ts = e->mono_time;
   }
-  postEvents();
+  emit privateUpdateLastMsgsSignal();
 }
 
 void LiveStream::seekTo(double sec) {

tools/cabana/streams/livestream.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include <deque>
 #include <memory>
 #include <vector>
 
@@ -15,6 +14,7 @@ public:
   LiveStream(QObject *parent);
   virtual ~LiveStream();
   void start() override;
+  inline QDateTime beginDateTime() const { return begin_date_time; }
   inline double routeStartTime() const override { return begin_event_ts / 1e9; }
   inline double currentSec() const override { return (current_event_ts - begin_event_ts) / 1e9; }
   void setSpeed(float speed) override { speed_ = speed; }
@@ -25,30 +25,21 @@ public:
 
 protected:
   virtual void streamThread() = 0;
-  void handleEvent(const char *data, const size_t size);
+  void handleEvent(kj::ArrayPtr<capnp::word> event);
 
 private:
   void startUpdateTimer();
   void timerEvent(QTimerEvent *event) override;
   void updateEvents();
 
-  struct Msg {
-    Msg(const char *data, const size_t size) {
-      event = ::new Event(aligned_buf.align(data, size));
-    }
-    ~Msg() { ::delete event; }
-    Event *event;
-    AlignedBuffer aligned_buf;
-  };
-
   std::mutex lock;
   QThread *stream_thread;
-  std::vector<Event *> receivedEvents;
+  std::vector<const CanEvent *> received_events_;
-  std::deque<Msg> receivedMessages;
 
   int timer_id;
   QBasicTimer update_timer;
 
+  QDateTime begin_date_time;
   uint64_t begin_event_ts = 0;
   uint64_t current_event_ts = 0;
   uint64_t first_event_ts = 0;

tools/cabana/streams/pandastream.cc

@@ -1,14 +1,13 @@
 #include "tools/cabana/streams/pandastream.h"
 
-#include <vector>
+#include <QDebug>
-
+#include <QCheckBox>
 #include <QLabel>
 #include <QMessageBox>
 #include <QPushButton>
+#include <QThread>
 #include <QVBoxLayout>
 
-#include "selfdrive/ui/qt/util.h"
-
 // TODO: remove clearLayout
 static void clearLayout(QLayout* layout) {
   while (layout->count() > 0) {
@@ -89,7 +88,6 @@ void PandaStream::streamThread() {
     MessageBuilder msg;
     auto evt = msg.initEvent();
     auto canData = evt.initCan(raw_can_data.size());
-
     for (uint i = 0; i<raw_can_data.size(); i++) {
       canData[i].setAddress(raw_can_data[i].address);
       canData[i].setBusTime(raw_can_data[i].busTime);
@@ -97,8 +95,7 @@ void PandaStream::streamThread() {
       canData[i].setSrc(raw_can_data[i].src);
     }
 
-    auto bytes = msg.toBytes();
+    handleEvent(capnp::messageToFlatArray(msg));
-    handleEvent((const char*)bytes.begin(), bytes.size());
 
     panda->send_heartbeat(false);
   }

tools/cabana/streams/pandastream.h

@@ -5,7 +5,6 @@
 
 #include <QComboBox>
 #include <QFormLayout>
-#include <QVBoxLayout>
 
 #include "tools/cabana/streams/livestream.h"
 #include "selfdrive/boardd/panda.h"

tools/cabana/streams/replaystream.cc

@@ -8,11 +8,14 @@
 
 ReplayStream::ReplayStream(QObject *parent) : AbstractStream(parent) {
   unsetenv("ZMQ");
+  setenv("COMMA_CACHE", "/tmp/comma_download_cache", 1);
+
   // TODO: Remove when OpenpilotPrefix supports ZMQ
 #ifndef __APPLE__
   op_prefix = std::make_unique<OpenpilotPrefix>();
 #endif
-  QObject::connect(&settings, &Settings::changed, [this]() {
+
+  QObject::connect(&settings, &Settings::changed, this, [this]() {
     if (replay) replay->setSegmentCacheLimit(settings.max_cached_minutes);
   });
 }
@@ -25,18 +28,30 @@ void ReplayStream::mergeSegments() {
   for (auto &[n, seg] : replay->segments()) {
     if (seg && seg->isLoaded() && !processed_segments.count(n)) {
       processed_segments.insert(n);
-      const auto &events = seg->log->events;
+
-      mergeEvents(events.cbegin(), events.cend());
+      std::vector<const CanEvent *> new_events;
+      new_events.reserve(seg->log->events.size());
+      for (auto it = seg->log->events.cbegin(); it != seg->log->events.cend(); ++it) {
+        if ((*it)->which == cereal::Event::Which::CAN) {
+          const uint64_t ts = (*it)->mono_time;
+          for (const auto &c : (*it)->event.getCan()) {
+            new_events.push_back(newEvent(ts, c));
+          }
+        }
+      }
+      mergeEvents(new_events);
     }
   }
 }
 
 bool ReplayStream::loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags) {
-  replay.reset(new Replay(route, {"can", "roadEncodeIdx", "wideRoadEncodeIdx", "carParams"}, {}, {}, nullptr, replay_flags, data_dir, this));
+  replay.reset(new Replay(route, {"can", "roadEncodeIdx", "driverEncodeIdx", "wideRoadEncodeIdx", "carParams"},
+                          {}, {}, nullptr, replay_flags, data_dir, this));
   replay->setSegmentCacheLimit(settings.max_cached_minutes);
   replay->installEventFilter(event_filter, this);
   QObject::connect(replay.get(), &Replay::seekedTo, this, &AbstractStream::seekedTo);
   QObject::connect(replay.get(), &Replay::segmentsMerged, this, &ReplayStream::mergeSegments);
+  QObject::connect(replay.get(), &Replay::qLogLoaded, this, &ReplayStream::qLogLoaded, Qt::QueuedConnection);
   return replay->load();
 }
 
@@ -47,7 +62,6 @@ void ReplayStream::start() {
 
 bool ReplayStream::eventFilter(const Event *event) {
   static double prev_update_ts = 0;
-  // delay posting CAN message if UI thread is busy
   if (event->which == cereal::Event::Which::CAN) {
     double current_sec = event->mono_time / 1e9 - routeStartTime();
     for (const auto &c : event->event.getCan()) {
@@ -59,9 +73,8 @@ bool ReplayStream::eventFilter(const Event *event) {
 
   double ts = millis_since_boot();
   if ((ts - prev_update_ts) > (1000.0 / settings.fps)) {
-    if (postEvents()) {
+    emit privateUpdateLastMsgsSignal();
-      prev_update_ts = ts;
+    prev_update_ts = ts;
-    }
   }
   return true;
 }
@@ -80,25 +93,21 @@ AbstractOpenStreamWidget *ReplayStream::widget(AbstractStream **stream) {
 
 OpenReplayWidget::OpenReplayWidget(AbstractStream **stream) : AbstractOpenStreamWidget(stream) {
   // TODO: get route list from api.comma.ai
-  QGridLayout *grid_layout = new QGridLayout();
+  QGridLayout *grid_layout = new QGridLayout(this);
   grid_layout->addWidget(new QLabel(tr("Route")), 0, 0);
   grid_layout->addWidget(route_edit = new QLineEdit(this), 0, 1);
   route_edit->setPlaceholderText(tr("Enter remote route name or click browse to select a local route"));
   auto file_btn = new QPushButton(tr("Browse..."), this);
   grid_layout->addWidget(file_btn, 0, 2);
 
-  grid_layout->addWidget(new QLabel(tr("Video")), 1, 0);
+  grid_layout->addWidget(new QLabel(tr("Camera")), 1, 0);
-  grid_layout->addWidget(choose_video_cb = new QComboBox(this), 1, 1);
+  QHBoxLayout *camera_layout = new QHBoxLayout();
-  QString items[] = {tr("No Video"), tr("Road Camera"), tr("Wide Road Camera"), tr("Driver Camera"), tr("QCamera")};
+  for (auto c : {tr("Road camera"), tr("Driver camera"), tr("Wide road camera")})
-  for (int i = 0; i < std::size(items); ++i) {
+    camera_layout->addWidget(cameras.emplace_back(new QCheckBox(c, this)));
-    choose_video_cb->addItem(items[i]);
+  camera_layout->addStretch(1);
-  }
+  grid_layout->addItem(camera_layout, 1, 1);
-  choose_video_cb->setCurrentIndex(1);  // default is road camera;
 
-  QVBoxLayout *main_layout = new QVBoxLayout(this);
-  main_layout->addLayout(grid_layout);
   setMinimumWidth(550);
-
   QObject::connect(file_btn, &QPushButton::clicked, [=]() {
     QString dir = QFileDialog::getExistingDirectory(this, tr("Open Local Route"), settings.last_route_dir);
     if (!dir.isEmpty()) {
@@ -120,9 +129,13 @@ bool OpenReplayWidget::open() {
   if (!is_valid_format) {
     QMessageBox::warning(nullptr, tr("Warning"), tr("Invalid route format: '%1'").arg(route));
   } else {
-    uint32_t flags[] = {REPLAY_FLAG_NO_VIPC, REPLAY_FLAG_NONE, REPLAY_FLAG_ECAM, REPLAY_FLAG_DCAM, REPLAY_FLAG_QCAMERA};
     auto replay_stream = std::make_unique<ReplayStream>(qApp);
-    if (replay_stream->loadRoute(route, data_dir, flags[choose_video_cb->currentIndex()])) {
+    uint32_t flags = REPLAY_FLAG_NONE;
+    if (cameras[1]->isChecked()) flags |= REPLAY_FLAG_DCAM;
+    if (cameras[2]->isChecked()) flags |= REPLAY_FLAG_ECAM;
+    if (flags == REPLAY_FLAG_NONE && !cameras[0]->isChecked()) flags = REPLAY_FLAG_NO_VIPC;
+
+    if (replay_stream->loadRoute(route, data_dir, flags)) {
       *stream = replay_stream.release();
     } else {
       QMessageBox::warning(nullptr, tr("Warning"), tr("Failed to load route: '%1'").arg(route));

tools/cabana/streams/replaystream.h

@@ -1,13 +1,14 @@
 #pragma once
 
+#include <QCheckBox>
 #include <algorithm>
 #include <memory>
 #include <set>
-#include <tuple>
 #include <vector>
 
 #include "common/prefix.h"
 #include "tools/cabana/streams/abstractstream.h"
+#include "tools/replay/replay.h"
 
 class ReplayStream : public AbstractStream {
   Q_OBJECT
@@ -18,20 +19,24 @@ public:
   bool loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags = REPLAY_FLAG_NONE);
   bool eventFilter(const Event *event);
   void seekTo(double ts) override { replay->seekTo(std::max(double(0), ts), false); }
+  bool liveStreaming() const override { return false; }
   inline QString routeName() const override { return replay->route()->name(); }
   inline QString carFingerprint() const override { return replay->carFingerprint().c_str(); }
   double totalSeconds() const override { return replay->totalSeconds(); }
-  inline VisionStreamType visionStreamType() const override { return replay->hasFlag(REPLAY_FLAG_ECAM) ? VISION_STREAM_WIDE_ROAD : VISION_STREAM_ROAD; }
+  inline QDateTime beginDateTime() const { return replay->route()->datetime(); }
   inline double routeStartTime() const override { return replay->routeStartTime() / (double)1e9; }
   inline double currentSec() const override { return replay->currentSeconds(); }
-  inline const Route *route() const override { return replay->route(); }
+  inline const Route *route() const { return replay->route(); }
   inline void setSpeed(float speed) override { replay->setSpeed(speed); }
   inline float getSpeed() const { return replay->getSpeed(); }
+  inline Replay *getReplay() const { return replay.get(); }
   inline bool isPaused() const override { return replay->isPaused(); }
   void pause(bool pause) override;
-  inline const std::vector<std::tuple<double, double, TimelineType>> getTimeline() override { return replay->getTimeline(); }
   static AbstractOpenStreamWidget *widget(AbstractStream **stream);
 
+signals:
+  void qLogLoaded(int segnum, std::shared_ptr<LogReader> qlog);
+
 private:
   void mergeSegments();
   std::unique_ptr<Replay> replay = nullptr;
@@ -49,5 +54,5 @@ public:
 
 private:
   QLineEdit *route_edit;
-  QComboBox *choose_video_cb;
+  std::vector<QCheckBox *> cameras;
 };

tools/cabana/streams/socketcanstream.cc

@@ -1,8 +1,11 @@
 #include "tools/cabana/streams/socketcanstream.h"
 
-#include <QLabel>
+#include <QDebug>
+#include <QFormLayout>
+#include <QHBoxLayout>
 #include <QMessageBox>
 #include <QPushButton>
+#include <QThread>
 
 SocketCanStream::SocketCanStream(QObject *parent, SocketCanStreamConfig config_) : config(config_), LiveStream(parent) {
   if (!available()) {
@@ -49,7 +52,6 @@ void SocketCanStream::streamThread() {
     auto evt = msg.initEvent();
     auto canData = evt.initCan(frames.size());
 
-
     for (uint i = 0; i < frames.size(); i++) {
       if (!frames[i].isValid()) continue;
 
@@ -60,8 +62,7 @@ void SocketCanStream::streamThread() {
       canData[i].setDat(kj::arrayPtr((uint8_t*)payload.data(), payload.size()));
     }
 
-    auto bytes = msg.toBytes();
+    handleEvent(capnp::messageToFlatArray(msg));
-    handleEvent((const char*)bytes.begin(), bytes.size());
   }
 }
 

tools/cabana/streams/socketcanstream.h

@@ -5,10 +5,7 @@
 #include <QtSerialBus/QCanBus>
 #include <QtSerialBus/QCanBusDevice>
 #include <QtSerialBus/QCanBusDeviceInfo>
-
 #include <QComboBox>
-#include <QFormLayout>
-#include <QVBoxLayout>
 
 #include "tools/cabana/streams/livestream.h"
 
@@ -21,7 +18,6 @@ class SocketCanStream : public LiveStream {
 public:
   SocketCanStream(QObject *parent, SocketCanStreamConfig config_ = {});
   static AbstractOpenStreamWidget *widget(AbstractStream **stream);
-
   static bool available();
 
   inline QString routeName() const override {

tools/cabana/streamselector.cc

@@ -2,7 +2,6 @@
 
 #include <QDialogButtonBox>
 #include <QFileDialog>
-#include <QFormLayout>
 #include <QLabel>
 #include <QPushButton>
 

tools/cabana/tools/findsignal.cc

@@ -192,7 +192,7 @@ void FindSignalDlg::search() {
   search_btn->setEnabled(false);
   stats_label->setVisible(false);
   search_btn->setText("Finding ....");
-  QTimer::singleShot(0, [=]() { model->search(cmp); });
+  QTimer::singleShot(0, this, [=]() { model->search(cmp); });
 }
 
 void FindSignalDlg::setInitialSignals() {
@@ -222,15 +222,15 @@ void FindSignalDlg::setInitialSignals() {
   }
   model->initial_signals.clear();
 
-  for (auto it = can->last_msgs.cbegin(); it != can->last_msgs.cend(); ++it) {
+  for (const auto &[id, m] : can->lastMessages()) {
-    if (buses.isEmpty() || buses.contains(it.key().source) && (addresses.isEmpty() || addresses.contains(it.key().address))) {
+    if (buses.isEmpty() || buses.contains(id.source) && (addresses.isEmpty() || addresses.contains(id.address))) {
-      const auto &events = can->events(it.key());
+      const auto &events = can->events(id);
       auto e = std::lower_bound(events.cbegin(), events.cend(), first_time, CompareCanEvent());
       if (e != events.cend()) {
-        const int total_size = it.value().dat.size() * 8;
+        const int total_size = m.dat.size() * 8;
         for (int size = min_size->value(); size <= max_size->value(); ++size) {
           for (int start = 0; start <= total_size - size; ++start) {
-            FindSignalModel::SearchSignal s{.id = it.key(), .mono_time = first_time, .sig = sig};
+            FindSignalModel::SearchSignal s{.id = id, .mono_time = first_time, .sig = sig};
             s.sig.start_bit = start;
             s.sig.size = size;
             updateMsbLsb(s.sig);

tools/cabana/tools/findsignal.h

@@ -4,6 +4,7 @@
 #include <limits>
 
 #include <QAbstractTableModel>
+#include <QCheckBox>
 #include <QLabel>
 #include <QPushButton>
 #include <QTableView>

tools/cabana/util.cc

@@ -9,7 +9,7 @@
 #include <unistd.h>
 
 #include <QColor>
-#include <QDebug>
+#include <QDateTime>
 #include <QFontDatabase>
 #include <QLocale>
 #include <QPainter>
@@ -19,7 +19,7 @@
 
 // SegmentTree
 
-void SegmentTree::build(const QVector<QPointF> &arr) {
+void SegmentTree::build(const std::vector<QPointF> &arr) {
   size = arr.size();
   tree.resize(4 * size);  // size of the tree is 4 times the size of the array
   if (size > 0) {
@@ -27,7 +27,7 @@ void SegmentTree::build(const QVector<QPointF> &arr) {
   }
 }
 
-void SegmentTree::build_tree(const QVector<QPointF> &arr, int n, int left, int right) {
+void SegmentTree::build_tree(const std::vector<QPointF> &arr, int n, int left, int right) {
   if (left == right) {
     const double y = arr[left].y();
     tree[n] = {y, y};
@@ -59,23 +59,18 @@ MessageBytesDelegate::MessageBytesDelegate(QObject *parent, bool multiple_lines)
     hex_text_table[i].setText(QStringLiteral("%1").arg(i, 2, 16, QLatin1Char('0')).toUpper());
     hex_text_table[i].prepare({}, fixed_font);
   }
+  h_margin = QApplication::style()->pixelMetric(QStyle::PM_FocusFrameHMargin) + 1;
+  v_margin = QApplication::style()->pixelMetric(QStyle::PM_FocusFrameVMargin) + 1;
 }
 
-int MessageBytesDelegate::widthForBytes(int n) const {
+QSize MessageBytesDelegate::sizeForBytes(int n) const {
-  int h_margin = QApplication::style()->pixelMetric(QStyle::PM_FocusFrameHMargin) + 1;
+  int rows = multiple_lines ? std::max(1, n / 8) : 1;
-  return n * byte_size.width() + h_margin * 2;
+  return {(n / rows) * byte_size.width() + h_margin * 2, rows * byte_size.height() + v_margin * 2};
 }
 
 QSize MessageBytesDelegate::sizeHint(const QStyleOptionViewItem &option, const QModelIndex &index) const {
-  int v_margin = QApplication::style()->pixelMetric(QStyle::PM_FocusFrameVMargin) + 1;
   auto data = index.data(BytesRole);
-  if (!data.isValid()) {
+  return sizeForBytes(data.isValid() ? static_cast<std::vector<uint8_t> *>(data.value<void *>())->size() : 0);
-    return {1, byte_size.height() + 2 * v_margin};
-  }
-  int n = data.toByteArray().size();
-  assert(n >= 0 && n <= 64);
-  return !multiple_lines ? QSize{widthForBytes(n), byte_size.height() + 2 * v_margin}
-                         : QSize{widthForBytes(8), byte_size.height() * std::max(1, n / 8) + 2 * v_margin};
 }
 
 void MessageBytesDelegate::paint(QPainter *painter, const QStyleOptionViewItem &option, const QModelIndex &index) const {
@@ -84,20 +79,17 @@ void MessageBytesDelegate::paint(QPainter *painter, const QStyleOptionViewItem &
     return QStyledItemDelegate::paint(painter, option, index);
   }
 
-  auto byte_list = data.toByteArray();
+  QFont old_font = painter->font();
-  auto colors = index.data(ColorsRole).value<QVector<QColor>>();
+  QPen old_pen = painter->pen();
-
-  int v_margin = option.widget->style()->pixelMetric(QStyle::PM_FocusFrameVMargin);
-  int h_margin = option.widget->style()->pixelMetric(QStyle::PM_FocusFrameHMargin);
   if (option.state & QStyle::State_Selected) {
     painter->fillRect(option.rect, option.palette.brush(QPalette::Normal, QPalette::Highlight));
   }
-
   const QPoint pt{option.rect.left() + h_margin, option.rect.top() + v_margin};
-  QFont old_font = painter->font();
-  QPen old_pen = painter->pen();
   painter->setFont(fixed_font);
-  for (int i = 0; i < byte_list.size(); ++i) {
+
+  const auto &bytes = *static_cast<std::vector<uint8_t>*>(data.value<void*>());
+  const auto &colors = *static_cast<std::vector<QColor>*>(index.data(ColorsRole).value<void*>());
+  for (int i = 0; i < bytes.size(); ++i) {
     int row = !multiple_lines ? 0 : i / 8;
     int column = !multiple_lines ? i : i % 8;
     QRect r = QRect({pt.x() + column * byte_size.width(), pt.y() + row * byte_size.height()}, byte_size);
@@ -110,7 +102,7 @@ void MessageBytesDelegate::paint(QPainter *painter, const QStyleOptionViewItem &
     } else if (option.state & QStyle::State_Selected) {
       painter->setPen(option.palette.color(QPalette::HighlightedText));
     }
-    utils::drawStaticText(painter, r, hex_text_table[(uint8_t)(byte_list[i])]);
+    utils::drawStaticText(painter, r, hex_text_table[bytes[i]]);
   }
   painter->setFont(old_font);
   painter->setPen(old_pen);
@@ -242,17 +234,15 @@ void setTheme(int theme) {
   }
 }
 
-}  // namespace utils
+QString formatSeconds(double sec, bool include_milliseconds, bool absolute_time) {
-
+  QString format = absolute_time ? "yyyy-MM-dd hh:mm:ss"
-QString toHex(uint8_t byte) {
+                                 : (sec > 60 * 60 ? "hh:mm:ss" : "mm:ss");
-  static std::array<QString, 256> hex = []() {
+  if (include_milliseconds) format += ".zzz";
-    std::array<QString, 256> ret;
+  return QDateTime::fromMSecsSinceEpoch(sec * 1000).toString(format);
-    for (int i = 0; i < 256; ++i) ret[i] = QStringLiteral("%1").arg(i, 2, 16, QLatin1Char('0')).toUpper();
-    return ret;
-  }();
-  return hex[byte];
 }
 
+}  // namespace utils
+
 int num_decimals(double num) {
   const QString string = QString::number(num);
   auto dot_pos = string.indexOf('.');

tools/cabana/util.h

@@ -8,7 +8,6 @@
 
 #include <QApplication>
 #include <QByteArray>
-#include <QDateTime>
 #include <QDoubleValidator>
 #include <QFont>
 #include <QPainter>
@@ -18,7 +17,6 @@
 #include <QStringBuilder>
 #include <QStyledItemDelegate>
 #include <QToolButton>
-#include <QVector>
 
 #include "tools/cabana/dbc/dbc.h"
 #include "tools/cabana/settings.h"
@@ -57,12 +55,12 @@ enum {
 class SegmentTree {
 public:
   SegmentTree() = default;
-  void build(const QVector<QPointF> &arr);
+  void build(const std::vector<QPointF> &arr);
   inline std::pair<double, double> minmax(int left, int right) const { return get_minmax(1, 0, size - 1, left, right); }
 
 private:
   std::pair<double, double> get_minmax(int n, int left, int right, int range_left, int range_right) const;
-  void build_tree(const QVector<QPointF> &arr, int n, int left, int right);
+  void build_tree(const std::vector<QPointF> &arr, int n, int left, int right);
   std::vector<std::pair<double, double>> tree;
   int size = 0;
 };
@@ -75,18 +73,16 @@ public:
   QSize sizeHint(const QStyleOptionViewItem &option, const QModelIndex &index) const override;
   bool multipleLines() const { return multiple_lines; }
   void setMultipleLines(bool v) { multiple_lines = v; }
-  int widthForBytes(int n) const;
+  QSize sizeForBytes(int n) const;
 
 private:
   std::array<QStaticText, 256> hex_text_table;
   QFont fixed_font;
   QSize byte_size = {};
   bool multiple_lines = false;
+  int h_margin, v_margin;
 };
 
-inline QString toHex(const QByteArray &dat) { return dat.toHex(' ').toUpper(); }
-QString toHex(uint8_t byte);
-
 class NameValidator : public QRegExpValidator {
   Q_OBJECT
 public:
@@ -103,13 +99,15 @@ public:
 namespace utils {
 QPixmap icon(const QString &id);
 void setTheme(int theme);
-inline QString formatSeconds(int seconds) {
+QString formatSeconds(double sec, bool include_milliseconds = false, bool absolute_time = false);
-  return QDateTime::fromSecsSinceEpoch(seconds, Qt::UTC).toString(seconds > 60 * 60 ? "hh:mm:ss" : "mm:ss");
-}
 inline void drawStaticText(QPainter *p, const QRect &r, const QStaticText &text) {
   auto size = (r.size() - text.size()) / 2;
   p->drawStaticText(r.left() + size.width(), r.top() + size.height(), text);
 }
+inline QString toHex(const std::vector<uint8_t> &dat, char separator = '\0') {
+  return QByteArray::fromRawData((const char *)dat.data(), dat.size()).toHex(separator).toUpper();
+}
+
 }
 
 class ToolButton : public QToolButton {

tools/cabana/videowidget.cc

@@ -1,20 +1,21 @@
 #include "tools/cabana/videowidget.h"
 
 #include <algorithm>
-#include <memory>
-#include <string>
 #include <utility>
 
-#include <QButtonGroup>
+#include <QAction>
-#include <QHBoxLayout>
+#include <QActionGroup>
+#include <QMenu>
 #include <QMouseEvent>
 #include <QPainter>
 #include <QStackedLayout>
 #include <QStyleOptionSlider>
-#include <QTimer>
 #include <QVBoxLayout>
 #include <QtConcurrent>
 
+#include "tools/cabana/streams/replaystream.h"
+#include "tools/cabana/util.h"
+
 const int MIN_VIDEO_HEIGHT = 100;
 const int THUMBNAIL_MARGIN = 3;
 
@@ -30,50 +31,16 @@ static const QColor timeline_colors[] = {
 VideoWidget::VideoWidget(QWidget *parent) : QFrame(parent) {
   setFrameStyle(QFrame::StyledPanel | QFrame::Plain);
   auto main_layout = new QVBoxLayout(this);
-  if (!can->liveStreaming()) {
+  if (!can->liveStreaming())
     main_layout->addWidget(createCameraWidget());
-  }
+  main_layout->addLayout(createPlaybackController());
-
-  // btn controls
-  QButtonGroup *group = new QButtonGroup(this);
-  group->setExclusive(true);
-
-  QHBoxLayout *control_layout = new QHBoxLayout();
-  play_btn = new QToolButton();
-  play_btn->setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Preferred);
-  control_layout->addWidget(play_btn);
-  if (can->liveStreaming()) {
-    control_layout->addWidget(skip_to_end_btn = new QToolButton(this));
-    skip_to_end_btn->setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Preferred);
-    skip_to_end_btn->setIcon(utils::icon("skip-end-fill"));
-    skip_to_end_btn->setToolTip(tr("Skip to the end"));
-    QObject::connect(skip_to_end_btn, &QToolButton::clicked, [group]() {
-      // set speed to 1.0
-      group->buttons()[2]->click();
-      can->pause(false);
-      can->seekTo(can->totalSeconds() + 1);
-    });
-  }
 
-  for (float speed : {0.1, 0.5, 1., 2.}) {
-    QToolButton *btn = new QToolButton(this);
-    btn->setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Preferred);
-    btn->setText(QString("%1x").arg(speed));
-    btn->setCheckable(true);
-    QObject::connect(btn, &QToolButton::clicked, [speed]() { can->setSpeed(speed); });
-    control_layout->addWidget(btn);
-    group->addButton(btn);
-    if (speed == 1.0) btn->setChecked(true);
-  }
-  main_layout->addLayout(control_layout);
   setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Maximum);
-
-  QObject::connect(play_btn, &QToolButton::clicked, []() { can->pause(!can->isPaused()); });
   QObject::connect(can, &AbstractStream::paused, this, &VideoWidget::updatePlayBtnState);
   QObject::connect(can, &AbstractStream::resume, this, &VideoWidget::updatePlayBtnState);
-  QObject::connect(&settings, &Settings::changed, this, &VideoWidget::updatePlayBtnState);
+  QObject::connect(can, &AbstractStream::msgsReceived, this, &VideoWidget::updateState);
-  updatePlayBtnState();
 
+  updatePlayBtnState();
   setWhatsThis(tr(R"(
     <b>Video</b><br />
     <!-- TODO: add descprition here -->
@@ -96,42 +63,138 @@ VideoWidget::VideoWidget(QWidget *parent) : QFrame(parent) {
           timeline_colors[(int)TimelineType::AlertCritical].name()));
 }
 
+QHBoxLayout *VideoWidget::createPlaybackController() {
+  QHBoxLayout *layout = new QHBoxLayout();
+  layout->addWidget(seek_backward_btn = new ToolButton("rewind", tr("Seek backward")));
+  layout->addWidget(play_btn = new ToolButton("play", tr("Play")));
+  layout->addWidget(seek_forward_btn = new ToolButton("fast-forward", tr("Seek forward")));
+
+  if (can->liveStreaming()) {
+    layout->addWidget(skip_to_end_btn = new ToolButton("skip-end", tr("Skip to the end"), this));
+    QObject::connect(skip_to_end_btn, &QToolButton::clicked, [this]() {
+      // set speed to 1.0
+      speed_btn->menu()->actions()[7]->setChecked(true);
+      can->pause(false);
+      can->seekTo(can->totalSeconds() + 1);
+    });
+  }
+
+  layout->addWidget(time_btn = new QToolButton);
+  time_btn->setToolTip(settings.absolute_time ? tr("Elapsed time") : tr("Absolute time"));
+  time_btn->setAutoRaise(true);
+  layout->addStretch(0);
+
+  if (!can->liveStreaming()) {
+    layout->addWidget(loop_btn = new ToolButton("repeat", tr("Loop playback")));
+    QObject::connect(loop_btn, &QToolButton::clicked, this, &VideoWidget::loopPlaybackClicked);
+  }
+
+  // speed selector
+  layout->addWidget(speed_btn = new QToolButton(this));
+  speed_btn->setAutoRaise(true);
+  speed_btn->setMenu(new QMenu(speed_btn));
+  speed_btn->setPopupMode(QToolButton::InstantPopup);
+  QActionGroup *speed_group = new QActionGroup(this);
+  speed_group->setExclusive(true);
+
+  int max_width = 0;
+  QFont font = speed_btn->font();
+  font.setBold(true);
+  speed_btn->setFont(font);
+  QFontMetrics fm(font);
+  for (float speed : {0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 0.8, 1., 2., 3., 5.}) {
+    QString name = QString("%1x").arg(speed);
+    max_width = std::max(max_width, fm.width(name) + fm.horizontalAdvance(QLatin1Char(' ')) * 2);
+
+    QAction *act = new QAction(name, speed_group);
+    act->setCheckable(true);
+    QObject::connect(act, &QAction::toggled, [this, speed]() {
+      can->setSpeed(speed);
+      speed_btn->setText(QString("%1x  ").arg(speed));
+    });
+    speed_btn->menu()->addAction(act);
+    if (speed == 1.0)act->setChecked(true);
+  }
+  speed_btn->setMinimumWidth(max_width + style()->pixelMetric(QStyle::PM_MenuButtonIndicator));
+
+  QObject::connect(play_btn, &QToolButton::clicked, []() { can->pause(!can->isPaused()); });
+  QObject::connect(seek_backward_btn, &QToolButton::clicked, []() { can->seekTo(can->currentSec() - 1); });
+  QObject::connect(seek_forward_btn, &QToolButton::clicked, []() { can->seekTo(can->currentSec() + 1); });
+  QObject::connect(time_btn, &QToolButton::clicked, [this]() {
+    settings.absolute_time = !settings.absolute_time;
+    time_btn->setToolTip(settings.absolute_time ? tr("Elapsed time") : tr("Absolute time"));
+    updateState();
+  });
+  return layout;
+}
+
 QWidget *VideoWidget::createCameraWidget() {
   QWidget *w = new QWidget(this);
   QVBoxLayout *l = new QVBoxLayout(w);
   l->setContentsMargins(0, 0, 0, 0);
+  l->setSpacing(0);
+
+  l->addWidget(camera_tab = new TabBar(w));
+  camera_tab->setAutoHide(true);
+  camera_tab->setExpanding(false);
 
   QStackedLayout *stacked = new QStackedLayout();
   stacked->setStackingMode(QStackedLayout::StackAll);
-  stacked->addWidget(cam_widget = new CameraWidget("camerad", can->visionStreamType(), false));
+  stacked->addWidget(cam_widget = new CameraWidget("camerad", VISION_STREAM_ROAD, false));
   cam_widget->setMinimumHeight(MIN_VIDEO_HEIGHT);
   cam_widget->setSizePolicy(QSizePolicy::Preferred, QSizePolicy::MinimumExpanding);
   stacked->addWidget(alert_label = new InfoLabel(this));
   l->addLayout(stacked);
 
-  // slider controls
+  l->addWidget(slider = new Slider(w));
-  auto slider_layout = new QHBoxLayout();
-  slider_layout->addWidget(time_label = new QLabel("00:00"));
-
-  slider = new Slider(this);
   slider->setSingleStep(0);
-  slider_layout->addWidget(slider);
-
-  slider_layout->addWidget(end_time_label = new QLabel(this));
-  l->addLayout(slider_layout);
 
   setMaximumTime(can->totalSeconds());
   QObject::connect(slider, &QSlider::sliderReleased, [this]() { can->seekTo(slider->currentSecond()); });
-  QObject::connect(slider, &QSlider::valueChanged, [=](int value) { time_label->setText(utils::formatSeconds(slider->currentSecond())); });
   QObject::connect(slider, &Slider::updateMaximumTime, this, &VideoWidget::setMaximumTime, Qt::QueuedConnection);
+  QObject::connect(static_cast<ReplayStream*>(can), &ReplayStream::qLogLoaded, slider, &Slider::parseQLog);
   QObject::connect(cam_widget, &CameraWidget::clicked, []() { can->pause(!can->isPaused()); });
-  QObject::connect(can, &AbstractStream::updated, this, &VideoWidget::updateState);
+  QObject::connect(cam_widget, &CameraWidget::vipcAvailableStreamsUpdated, this, &VideoWidget::vipcAvailableStreamsUpdated);
+  QObject::connect(camera_tab, &QTabBar::currentChanged, [this](int index) {
+    if (index != -1) cam_widget->setStreamType((VisionStreamType)camera_tab->tabData(index).toInt());
+  });
   return w;
 }
 
+void VideoWidget::vipcAvailableStreamsUpdated(std::set<VisionStreamType> streams) {
+  static const QString stream_names[] = {
+    [VISION_STREAM_ROAD] = "Road camera",
+    [VISION_STREAM_WIDE_ROAD] = "Wide road camera",
+    [VISION_STREAM_DRIVER] = "Driver camera"};
+
+  for (int i = 0; i < streams.size(); ++i) {
+    if (camera_tab->count() <= i) {
+      camera_tab->addTab(QString());
+    }
+    int type = *std::next(streams.begin(), i);
+    camera_tab->setTabText(i, stream_names[type]);
+    camera_tab->setTabData(i, type);
+  }
+  while (camera_tab->count() > streams.size()) {
+    camera_tab->removeTab(camera_tab->count() - 1);
+  }
+}
+
+void VideoWidget::loopPlaybackClicked() {
+  auto replay = qobject_cast<ReplayStream *>(can)->getReplay();
+  if (!replay) return;
+
+  if (replay->hasFlag(REPLAY_FLAG_NO_LOOP)) {
+    replay->removeFlag(REPLAY_FLAG_NO_LOOP);
+    loop_btn->setIcon("repeat");
+  } else {
+    replay->addFlag(REPLAY_FLAG_NO_LOOP);
+    loop_btn->setIcon("repeat-1");
+  }
+}
+
 void VideoWidget::setMaximumTime(double sec) {
   maximum_time = sec;
-  end_time_label->setText(utils::formatSeconds(sec));
   slider->setTimeRange(0, sec);
 }
 
@@ -140,24 +203,30 @@ void VideoWidget::updateTimeRange(double min, double max, bool is_zoomed) {
     skip_to_end_btn->setEnabled(!is_zoomed);
     return;
   }
+  is_zoomed ? slider->setTimeRange(min, max)
+            : slider->setTimeRange(0, maximum_time);
+}
 
-  if (!is_zoomed) {
+QString VideoWidget::formatTime(double sec, bool include_milliseconds) {
-    min = 0;
+  if (settings.absolute_time)
-    max = maximum_time;
+    sec = can->beginDateTime().addMSecs(sec * 1000).toMSecsSinceEpoch() / 1000.0;
-  }
+  return utils::formatSeconds(sec, include_milliseconds, settings.absolute_time);
-  end_time_label->setText(utils::formatSeconds(max));
-  slider->setTimeRange(min, max);
 }
 
 void VideoWidget::updateState() {
-  if (!slider->isSliderDown()) {
+  if (slider) {
-    slider->setCurrentSecond(can->currentSec());
+    if (!slider->isSliderDown())
+      slider->setCurrentSecond(can->currentSec());
+    alert_label->showAlert(slider->alertInfo(can->currentSec()));
+    time_btn->setText(QString("%1 / %2").arg(formatTime(can->currentSec(), true),
+                                             formatTime(slider->maximum() / slider->factor)));
+  } else {
+    time_btn->setText(formatTime(can->currentSec(), true));
   }
-  alert_label->showAlert(slider->alertInfo(can->currentSec()));
 }
 
 void VideoWidget::updatePlayBtnState() {
-  play_btn->setIcon(utils::icon(can->isPaused() ? "play" : "pause"));
+  play_btn->setIcon(can->isPaused() ? "play" : "pause");
   play_btn->setToolTip(can->isPaused() ? tr("Play") : tr("Pause"));
 }
 
@@ -165,29 +234,9 @@ void VideoWidget::updatePlayBtnState() {
 
 Slider::Slider(QWidget *parent) : thumbnail_label(parent), QSlider(Qt::Horizontal, parent) {
   setMouseTracking(true);
-  auto timer = new QTimer(this);
-  timer->callOnTimeout([this]() {
-    timeline = can->getTimeline();
-    std::sort(timeline.begin(), timeline.end(), [](auto &l, auto &r) { return std::get<2>(l) < std::get<2>(r); });
-    update();
-  });
-  timer->start(2000);
-  QObject::connect(can, &AbstractStream::eventsMerged, [this]() {
-    if (!qlog_future) {
-      qlog_future = std::make_unique<QFuture<void>>(QtConcurrent::run(this, &Slider::parseQLog));
-    }
-  });
-  QObject::connect(qApp, &QApplication::aboutToQuit, [this]() {
-    abort_parse_qlog = true;
-    if (qlog_future && qlog_future->isRunning()) {
-      qDebug() << "stopping thumbnail thread";
-      qlog_future->waitForFinished();
-    }
-  });
 }
 
 AlertInfo Slider::alertInfo(double seconds) {
-  std::lock_guard lk(thumbnail_lock);
   uint64_t mono_time = (seconds + can->routeStartTime()) * 1e9;
   auto alert_it = alerts.lower_bound(mono_time);
   bool has_alert = (alert_it != alerts.end()) && ((alert_it->first - mono_time) <= 1e8);
@@ -195,7 +244,6 @@ AlertInfo Slider::alertInfo(double seconds) {
 }
 
 QPixmap Slider::thumbnail(double seconds)  {
-  std::lock_guard lk(thumbnail_lock);
   uint64_t mono_time = (seconds + can->routeStartTime()) * 1e9;
   auto it = thumbnails.lowerBound(mono_time);
   return it != thumbnails.end() ? it.value() : QPixmap();
@@ -206,36 +254,32 @@ void Slider::setTimeRange(double min, double max) {
   setRange(min * factor, max * factor);
 }
 
-void Slider::parseQLog() {
+void Slider::parseQLog(int segnum, std::shared_ptr<LogReader> qlog) {
-  const auto &segments = can->route()->segments();
+ const auto &segments = qobject_cast<ReplayStream *>(can)->route()->segments();
-  for (auto it = segments.rbegin(); it != segments.rend() && !abort_parse_qlog; ++it) {
+  if (segments.size() > 0 && segnum == segments.rbegin()->first && !qlog->events.empty()) {
-    LogReader log;
+    emit updateMaximumTime(qlog->events.back()->mono_time / 1e9 - can->routeStartTime());
-    std::string qlog = it->second.qlog.toStdString();
+  }
-    if (!qlog.empty() && log.load(qlog, &abort_parse_qlog, {cereal::Event::Which::THUMBNAIL, cereal::Event::Which::CONTROLS_STATE}, true, 0, 3)) {
+
-      if (it == segments.rbegin() && !log.events.empty()) {
+  std::mutex mutex;
-        double max_time = log.events.back()->mono_time / 1e9 - can->routeStartTime();
+  QtConcurrent::blockingMap(qlog->events.cbegin(), qlog->events.cend(), [&mutex, this](const Event *e) {
-        emit updateMaximumTime(max_time);
+    if (e->which == cereal::Event::Which::THUMBNAIL) {
+      auto thumb = e->event.getThumbnail();
+      auto data = thumb.getThumbnail();
+      if (QPixmap pm; pm.loadFromData(data.begin(), data.size(), "jpeg")) {
+        QPixmap scaled = pm.scaledToHeight(MIN_VIDEO_HEIGHT - THUMBNAIL_MARGIN * 2, Qt::SmoothTransformation);
+        std::lock_guard lk(mutex);
+        thumbnails[thumb.getTimestampEof()] = scaled;
       }
-      for (auto ev = log.events.cbegin(); ev != log.events.cend() && !abort_parse_qlog; ++ev) {
+    } else if (e->which == cereal::Event::Which::CONTROLS_STATE) {
-        if ((*ev)->which == cereal::Event::Which::THUMBNAIL) {
+      auto cs = e->event.getControlsState();
-          auto thumb = (*ev)->event.getThumbnail();
+      if (cs.getAlertType().size() > 0 && cs.getAlertText1().size() > 0 &&
-          auto data = thumb.getThumbnail();
+          cs.getAlertSize() != cereal::ControlsState::AlertSize::NONE) {
-          if (QPixmap pm; pm.loadFromData(data.begin(), data.size(), "jpeg")) {
+        std::lock_guard lk(mutex);
-            pm = pm.scaledToHeight(MIN_VIDEO_HEIGHT - THUMBNAIL_MARGIN * 2, Qt::SmoothTransformation);
+        alerts.emplace(e->mono_time, AlertInfo{cs.getAlertStatus(), cs.getAlertText1().cStr(), cs.getAlertText2().cStr()});
-            std::lock_guard lk(thumbnail_lock);
-            thumbnails[thumb.getTimestampEof()] = pm;
-          }
-        } else if ((*ev)->which == cereal::Event::Which::CONTROLS_STATE) {
-          auto cs = (*ev)->event.getControlsState();
-          if (cs.getAlertType().size() > 0 && cs.getAlertText1().size() > 0 &&
-              cs.getAlertSize() != cereal::ControlsState::AlertSize::NONE) {
-            std::lock_guard lk(thumbnail_lock);
-            alerts.emplace((*ev)->mono_time, AlertInfo{cs.getAlertStatus(), cs.getAlertText1().cStr(), cs.getAlertText2().cStr()});
-          }
-        }
       }
     }
-  }
+  });
+  update();
 }
 
 void Slider::paintEvent(QPaintEvent *ev) {
@@ -245,7 +289,7 @@ void Slider::paintEvent(QPaintEvent *ev) {
   double min = minimum() / factor;
   double max = maximum() / factor;
 
-  for (auto [begin, end, type] : timeline) {
+  for (auto [begin, end, type] : qobject_cast<ReplayStream *>(can)->getReplay()->getTimeline()) {
     if (begin > max || end < min)
       continue;
     r.setLeft(((std::max(min, begin) - min) / (max - min)) * width());
@@ -265,8 +309,7 @@ void Slider::paintEvent(QPaintEvent *ev) {
 void Slider::mousePressEvent(QMouseEvent *e) {
   QSlider::mousePressEvent(e);
   if (e->button() == Qt::LeftButton && !isSliderDown()) {
-    int value = minimum() + ((maximum() - minimum()) * e->x()) / width();
+    setValue(minimum() + ((maximum() - minimum()) * e->x()) / width());
-    setValue(value);
     emit sliderReleased();
   }
 }
@@ -276,9 +319,9 @@ void Slider::mouseMoveEvent(QMouseEvent *e) {
   double seconds = (minimum() + pos * ((maximum() - minimum()) / (double)width())) / factor;
   QPixmap thumb = thumbnail(seconds);
   if (!thumb.isNull()) {
-    int x = std::clamp(pos - thumb.width() / 2, THUMBNAIL_MARGIN, rect().right() - thumb.width() - THUMBNAIL_MARGIN);
+    int x = std::clamp(pos - thumb.width() / 2, THUMBNAIL_MARGIN, width() - thumb.width() - THUMBNAIL_MARGIN + 1);
-    int y = -thumb.height();
+    int y = -thumb.height() - THUMBNAIL_MARGIN;
-    thumbnail_label.showPixmap(mapToParent({x, y}), utils::formatSeconds(seconds), thumb, alertInfo(seconds));
+    thumbnail_label.showPixmap(mapToParent(QPoint(x, y)), utils::formatSeconds(seconds), thumb, alertInfo(seconds));
   } else {
     thumbnail_label.hide();
   }
@@ -311,8 +354,7 @@ void InfoLabel::showPixmap(const QPoint &pt, const QString &sec, const QPixmap &
   second = sec;
   pixmap = pm;
   alert_info = alert;
-  resize(pm.size());
+  setGeometry(QRect(pt, pm.size()));
-  move(pt);
   setVisible(true);
   update();
 }

tools/cabana/videowidget.h

@@ -1,19 +1,18 @@
 #pragma once
 
-#include <atomic>
 #include <map>
 #include <memory>
-#include <mutex>
+#include <set>
-#include <tuple>
-#include <vector>
 
-#include <QFuture>
+#include <QHBoxLayout>
-#include <QLabel>
+#include <QFrame>
 #include <QSlider>
+#include <QTabBar>
 #include <QToolButton>
 
 #include "selfdrive/ui/qt/widgets/cameraview.h"
-#include "tools/cabana/streams/abstractstream.h"
+#include "tools/cabana/util.h"
+#include "tools/replay/logreader.h"
 
 struct AlertInfo {
   cereal::ControlsState::AlertStatus status;
@@ -42,6 +41,9 @@ public:
   void setTimeRange(double min, double max);
   AlertInfo alertInfo(double sec);
   QPixmap thumbnail(double sec);
+  void parseQLog(int segnum, std::shared_ptr<LogReader> qlog);
+
+  const double factor = 1000.0;
 
 signals:
   void updateMaximumTime(double);
@@ -51,15 +53,9 @@ private:
   void mouseMoveEvent(QMouseEvent *e) override;
   bool event(QEvent *event) override;
   void paintEvent(QPaintEvent *ev) override;
-  void parseQLog();
 
-  const double factor = 1000.0;
-  std::vector<std::tuple<double, double, TimelineType>> timeline;
-  std::mutex thumbnail_lock;
-  std::atomic<bool> abort_parse_qlog = false;
   QMap<uint64_t, QPixmap> thumbnails;
   std::map<uint64_t, AlertInfo> alerts;
-  std::unique_ptr<QFuture<void>> qlog_future;
   InfoLabel thumbnail_label;
 };
 
@@ -72,16 +68,24 @@ public:
   void setMaximumTime(double sec);
 
 protected:
+  QString formatTime(double sec, bool include_milliseconds = false);
   void updateState();
   void updatePlayBtnState();
   QWidget *createCameraWidget();
+  QHBoxLayout *createPlaybackController();
+  void loopPlaybackClicked();
+  void vipcAvailableStreamsUpdated(std::set<VisionStreamType> streams);
 
   CameraWidget *cam_widget;
   double maximum_time = 0;
-  QLabel *end_time_label;
+  QToolButton *time_btn = nullptr;
-  QLabel *time_label;
+  ToolButton *seek_backward_btn = nullptr;
-  QToolButton *play_btn;
+  ToolButton *play_btn = nullptr;
-  QToolButton *skip_to_end_btn = nullptr;
+  ToolButton *seek_forward_btn = nullptr;
-  InfoLabel *alert_label;
+  ToolButton *loop_btn = nullptr;
-  Slider *slider;
+  QToolButton *speed_btn = nullptr;
+  ToolButton *skip_to_end_btn = nullptr;
+  InfoLabel *alert_label = nullptr;
+  Slider *slider = nullptr;
+  QTabBar *camera_tab = nullptr;
 };

tools/lib/url_file.py

@@ -17,6 +17,10 @@ def hash_256(link):
   return hsh
 
 
+class URLFileException(Exception):
+  pass
+
+
 class URLFile:
   _tlocal = threading.local()
 
@@ -158,11 +162,11 @@ class URLFile:
 
     response_code = c.getinfo(pycurl.RESPONSE_CODE)
     if response_code == 416:  # Requested Range Not Satisfiable
-      raise Exception(f"Error, range out of bounds {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}")
+      raise URLFileException(f"Error, range out of bounds {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}")
     if download_range and response_code != 206:  # Partial Content
-      raise Exception(f"Error, requested range but got unexpected response {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}")
+      raise URLFileException(f"Error, requested range but got unexpected response {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}")
     if (not download_range) and response_code != 200:  # OK
-      raise Exception(f"Error {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}")
+      raise URLFileException(f"Error {response_code} {headers} ({self._url}): {repr(dats.getvalue())[:500]}")
 
     ret = dats.getvalue()
     self._pos += len(ret)

tools/replay/replay.cc

@@ -153,12 +153,10 @@ void Replay::buildTimeline() {
 
   const auto &route_segments = route_->segments();
   for (auto it = route_segments.cbegin(); it != route_segments.cend() && !exit_; ++it) {
-    LogReader log;
+    std::shared_ptr<LogReader> log(new LogReader());
-    if (!log.load(it->second.qlog.toStdString(), &exit_,
+    if (!log->load(it->second.qlog.toStdString(), &exit_, {}, !hasFlag(REPLAY_FLAG_NO_FILE_CACHE), 0, 3)) continue;
-                  {cereal::Event::Which::CONTROLS_STATE, cereal::Event::Which::USER_FLAG},
-                  !hasFlag(REPLAY_FLAG_NO_FILE_CACHE), 0, 3)) continue;
 
-    for (const Event *e : log.events) {
+    for (const Event *e : log->events) {
       if (e->which == cereal::Event::Which::CONTROLS_STATE) {
         auto cs = e->event.getControlsState();
 
@@ -186,6 +184,8 @@ void Replay::buildTimeline() {
         timeline.push_back({toSeconds(e->mono_time), toSeconds(e->mono_time), TimelineType::UserFlag});
       }
     }
+    std::sort(timeline.begin(), timeline.end(), [](auto &l, auto &r) { return std::get<2>(l) < std::get<2>(r); });
+    emit qLogLoaded(it->first, log);
   }
 }
 

tools/replay/replay.h

@@ -44,6 +44,7 @@ enum class FindFlag {
 
 enum class TimelineType { None, Engaged, AlertInfo, AlertWarning, AlertCritical, UserFlag };
 typedef bool (*replayEventFilter)(const Event *, void *);
+Q_DECLARE_METATYPE(std::shared_ptr<LogReader>);
 
 class Replay : public QObject {
   Q_OBJECT
@@ -91,6 +92,7 @@ signals:
   void streamStarted();
   void segmentsMerged();
   void seekedTo(double sec);
+  void qLogLoaded(int segnum, std::shared_ptr<LogReader> qlog);
 
 protected slots:
   void segmentLoadFinished(bool success);