From 4d7b7483d70a61cf0fb938107df55c24f27da1b0 Mon Sep 17 00:00:00 2001
From: Gijs Koning <gijs-koning@live.nl>
Date: Tue, 12 Jul 2022 12:36:57 +0200
Subject: [PATCH] Replay: tolerances per field (#25116)

* tolerances per field in replay

* refactor

* Remove laikad parameters

* Small comment change
---
 selfdrive/test/process_replay/compare_logs.py | 22 ++++++++++++++++---
 .../test/process_replay/process_replay.py     |  2 +-
 .../test/process_replay/test_processes.py     |  2 +-
 3 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/selfdrive/test/process_replay/compare_logs.py b/selfdrive/test/process_replay/compare_logs.py
index 057e46cd9c..bf6daf5fed 100755
--- a/selfdrive/test/process_replay/compare_logs.py
+++ b/selfdrive/test/process_replay/compare_logs.py
@@ -46,11 +46,25 @@ def remove_ignored_fields(msg, ignore):
   return msg.as_reader()
 
 
-def compare_logs(log1, log2, ignore_fields=None, ignore_msgs=None, tolerance=None):
+def get_field_tolerance(diff_field, field_tolerances):
+  diff_field_str = diff_field[0]
+  for s in diff_field[1:]:
+    # loop until number in field
+    if not isinstance(s, str):
+      break
+    diff_field_str += '.'+s
+  if diff_field_str in field_tolerances:
+    return field_tolerances[diff_field_str]
+
+
+def compare_logs(log1, log2, ignore_fields=None, ignore_msgs=None, tolerance=None, field_tolerances=None):
   if ignore_fields is None:
     ignore_fields = []
   if ignore_msgs is None:
     ignore_msgs = []
+  if field_tolerances is None:
+    field_tolerances = {}
+  default_tolerance = EPSILON if tolerance is None else tolerance
 
   log1, log2 = (list(filter(lambda m: m.which() not in ignore_msgs, log)) for log in (log1, log2))
 
@@ -72,7 +86,6 @@ def compare_logs(log1, log2, ignore_fields=None, ignore_msgs=None, tolerance=Non
       msg1_dict = msg1.to_dict(verbose=True)
       msg2_dict = msg2.to_dict(verbose=True)
 
-      tolerance = EPSILON if tolerance is None else tolerance
       dd = dictdiffer.diff(msg1_dict, msg2_dict, ignore=ignore_fields)
 
       # Dictdiffer only supports relative tolerance, we also want to check for absolute
@@ -80,10 +93,13 @@ def compare_logs(log1, log2, ignore_fields=None, ignore_msgs=None, tolerance=Non
       def outside_tolerance(diff):
         try:
           if diff[0] == "change":
+            field_tolerance = default_tolerance
+            if (tol := get_field_tolerance(diff[1], field_tolerances)) is not None:
+              field_tolerance = tol
             a, b = diff[2]
             finite = math.isfinite(a) and math.isfinite(b)
             if finite and isinstance(a, numbers.Number) and isinstance(b, numbers.Number):
-              return abs(a - b) > max(tolerance, tolerance * max(abs(a), abs(b)))
+              return abs(a - b) > max(field_tolerance, field_tolerance * max(abs(a), abs(b)))
         except TypeError:
           pass
         return True
diff --git a/selfdrive/test/process_replay/process_replay.py b/selfdrive/test/process_replay/process_replay.py
index bea7dc46ee..0c642cde17 100755
--- a/selfdrive/test/process_replay/process_replay.py
+++ b/selfdrive/test/process_replay/process_replay.py
@@ -28,7 +28,7 @@ TIMEOUT = 15
 PROC_REPLAY_DIR = os.path.dirname(os.path.abspath(__file__))
 FAKEDATA = os.path.join(PROC_REPLAY_DIR, "fakedata/")
 
-ProcessConfig = namedtuple('ProcessConfig', ['proc_name', 'pub_sub', 'ignore', 'init_callback', 'should_recv_callback', 'tolerance', 'fake_pubsubmaster', 'submaster_config', 'environ', 'subtest_name'], defaults=({}, {}, ""))
+ProcessConfig = namedtuple('ProcessConfig', ['proc_name', 'pub_sub', 'ignore', 'init_callback', 'should_recv_callback', 'tolerance', 'fake_pubsubmaster', 'submaster_config', 'environ', 'subtest_name', "field_tolerances"], defaults=({}, {}, "", {}))
 
 
 def wait_for_event(evt):
diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py
index 91cc40f5ce..d8cd1fd57a 100755
--- a/selfdrive/test/process_replay/test_processes.py
+++ b/selfdrive/test/process_replay/test_processes.py
@@ -100,7 +100,7 @@ def test_process(cfg, lr, ref_log_path, new_log_path, ignore_fields=None, ignore
       return f"Route did not enable at all or for long enough: {new_log_path}", log_msgs
 
   try:
-    return compare_logs(ref_log_msgs, log_msgs, ignore_fields + cfg.ignore, ignore_msgs, cfg.tolerance), log_msgs
+    return compare_logs(ref_log_msgs, log_msgs, ignore_fields + cfg.ignore, ignore_msgs, cfg.tolerance, cfg.field_tolerances), log_msgs
   except Exception as e:
     return str(e), log_msgs