From ba7039785c050bb880d7da7f32c303c9d6a89c29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Harald=20Sch=C3=A4fer?= <harald.the.engineer@gmail.com>
Date: Wed, 25 Sep 2024 14:21:19 -0700
Subject: [PATCH] More reasonable model replay tolerance (#33652)

---
 selfdrive/test/process_replay/model_replay.py | 28 ++++++++++++-------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/selfdrive/test/process_replay/model_replay.py b/selfdrive/test/process_replay/model_replay.py
index bb7c0b0a8e..269cf464a3 100755
--- a/selfdrive/test/process_replay/model_replay.py
+++ b/selfdrive/test/process_replay/model_replay.py
@@ -112,16 +112,24 @@ if __name__ == "__main__":
         'driverStateV2.dspExecutionTime'
       ]
       if PC:
-        ignore += [
-          'modelV2.laneLines.0.t',
-          'modelV2.laneLines.1.t',
-          'modelV2.laneLines.2.t',
-          'modelV2.laneLines.3.t',
-          'modelV2.roadEdges.0.t',
-          'modelV2.roadEdges.1.t',
-        ]
-      # TODO this tolerance is absurdly large
-      tolerance = 2.0 if PC else None
+        # TODO We ignore whole bunch so we can compare important stuff
+        # like posenet with reasonable tolerance
+        ignore += ['modelV2.acceleration.x',
+                   'modelV2.position.x',
+                   'modelV2.position.xStd',
+                   'modelV2.position.y',
+                   'modelV2.position.yStd',
+                   'modelV2.position.z',
+                   'modelV2.position.zStd',
+                   'drivingModelData.path.xCoefficients',]
+        for i in range(3):
+          for field in ('x', 'y', 'v', 'a'):
+            ignore.append(f'modelV2.leadsV3.{i}.{field}')
+            ignore.append(f'modelV2.leadsV3.{i}.{field}Std')
+        for i in range(4):
+          for field in ('x', 'y', 'z', 't'):
+            ignore.append(f'modelV2.laneLines.{i}.{field}')
+      tolerance = .2 if PC else None
       results: Any = {TEST_ROUTE: {}}
       log_paths: Any = {TEST_ROUTE: {"models": {'ref': BASE_URL + log_fn, 'new': log_fn}}}
       results[TEST_ROUTE]["models"] = compare_logs(cmp_log, log_msgs, tolerance=tolerance, ignore_fields=ignore)