From 7e716dbc87c28eb519dbb74bc76435f07edc408f Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Tue, 15 Aug 2023 20:06:15 -0700
Subject: [PATCH] CANParser: perf benchmark on route script (#29324)

* timing test

* test

* fix

* print

* loop

* clean up

* wider range

* Update selfdrive/car/tests/test_models.py

* Apply suggestions from code review

* run many times

* Update selfdrive/car/tests/test_models.py

* run for many its

* run with unittest and print as list

* Update .github/workflows/selfdrive_tests.yaml

* Update .github/workflows/selfdrive_tests.yaml

* total time is super inconsistent (body)

* Update selfdrive/car/tests/test_models.py

* clean up

* clean up

* clean up

* this works!

* draft

* test suite not as modular

* try something like this

* can do kb, but not too representative

* clean up

* remove kb? it depends on signals

* clean up

* more clean up

* rename

* just measure all CANParsers

* can do all this manually

* but this is way simpler

* comment

* fix mypy

* this is correct too

* 3 seconds instead of 1.5
old-commit-hash: e2910d0720f0b65496234382847da1ab0d086e4d
---
 selfdrive/car/tests/test_models.py            |  3 ++
 .../debug/check_can_parser_performance.py     | 37 +++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100755 selfdrive/debug/check_can_parser_performance.py

diff --git a/selfdrive/car/tests/test_models.py b/selfdrive/car/tests/test_models.py
index b11b067350..47ca2bbf08 100755
--- a/selfdrive/car/tests/test_models.py
+++ b/selfdrive/car/tests/test_models.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 # pylint: disable=E1101
+import capnp
 import os
 import importlib
 import unittest
@@ -70,6 +71,8 @@ class TestCarModelBase(unittest.TestCase):
   test_route: Optional[CarTestRoute] = None
   ci: bool = True
 
+  can_msgs: List[capnp.lib.capnp._DynamicStructReader]
+
   @unittest.skipIf(SKIP_ENV_VAR in os.environ, f"Long running test skipped. Unset {SKIP_ENV_VAR} to run")
   @classmethod
   def setUpClass(cls):
diff --git a/selfdrive/debug/check_can_parser_performance.py b/selfdrive/debug/check_can_parser_performance.py
new file mode 100755
index 0000000000..4b430e013f
--- /dev/null
+++ b/selfdrive/debug/check_can_parser_performance.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+import numpy as np
+import time
+from tqdm import tqdm
+
+from cereal import car
+from selfdrive.car.tests.routes import CarTestRoute
+from selfdrive.car.tests.test_models import TestCarModelBase
+from tools.plotjuggler.juggle import DEMO_ROUTE
+
+N_RUNS = 10
+
+
+class CarModelTestCase(TestCarModelBase):
+  test_route = CarTestRoute(DEMO_ROUTE, None)
+  ci = False
+
+
+if __name__ == '__main__':
+  # Get CAN messages and parsers
+  tm = CarModelTestCase()
+  tm.setUpClass()
+  tm.setUp()
+
+  CC = car.CarControl.new_message()
+  ets = []
+  for _ in tqdm(range(N_RUNS)):
+    start_t = time.process_time_ns()
+    for msg in tm.can_msgs:
+      for cp in tm.CI.can_parsers:
+        if cp is not None:
+          cp.update_strings((msg.as_builder().to_bytes(),))
+    ets.append((time.process_time_ns() - start_t) * 1e-6)
+
+  print(f'{len(tm.can_msgs)} CAN packets, {N_RUNS} runs')
+  print(f'{np.mean(ets):.2f} mean ms, {max(ets):.2f} max ms, {min(ets):.2f} min ms, {np.std(ets):.2f} std ms')
+  print(f'{np.mean(ets) / len(tm.can_msgs):.4f} mean ms / CAN packet')