diff --git a/selfdrive/test/process_replay/compare_logs.py b/selfdrive/test/process_replay/compare_logs.py index 7de0a25761..ce8cd2864e 100755 --- a/selfdrive/test/process_replay/compare_logs.py +++ b/selfdrive/test/process_replay/compare_logs.py @@ -5,6 +5,7 @@ import capnp import numbers import dictdiffer from collections import Counter +from typing import Dict from openpilot.tools.lib.logreader import LogReader @@ -89,7 +90,52 @@ def compare_logs(log1, log2, ignore_fields=None, ignore_msgs=None, tolerance=Non return diff +def format_diff(results, log_paths, ref_commit): + diff1, diff2 = "", "" + diff2 += f"***** tested against commit {ref_commit} *****\n" + + failed = False + for segment, result in list(results.items()): + diff1 += f"***** results for segment {segment} *****\n" + diff2 += f"***** differences for segment {segment} *****\n" + + for proc, diff in list(result.items()): + # long diff + diff2 += f"*** process: {proc} ***\n" + diff2 += f"\tref: {log_paths[segment][proc]['ref']}\n" + diff2 += f"\tnew: {log_paths[segment][proc]['new']}\n\n" + + # short diff + diff1 += f" {proc}\n" + if isinstance(diff, str): + diff1 += f" ref: {log_paths[segment][proc]['ref']}\n" + diff1 += f" new: {log_paths[segment][proc]['new']}\n\n" + diff1 += f" {diff}\n" + failed = True + elif len(diff): + diff1 += f" ref: {log_paths[segment][proc]['ref']}\n" + diff1 += f" new: {log_paths[segment][proc]['new']}\n\n" + + cnt: Dict[str, int] = {} + for d in diff: + diff2 += f"\t{str(d)}\n" + + k = str(d[1]) + cnt[k] = 1 if k not in cnt else cnt[k] + 1 + + for k, v in sorted(cnt.items()): + diff1 += f" {k}: {v}\n" + failed = True + return diff1, diff2, failed + + if __name__ == "__main__": log1 = list(LogReader(sys.argv[1])) log2 = list(LogReader(sys.argv[2])) - print(compare_logs(log1, log2, sys.argv[3:])) + ignore_fields = sys.argv[3:] or ["logMonoTime", "controlsState.startMonoTime", "controlsState.cumLagMs"] + results = {"segment": {"proc": compare_logs(log1, log2, ignore_fields)}} + log_paths = {"segment": {"proc": {"ref": sys.argv[1], "new": sys.argv[2]}}} + diff1, diff2, failed = format_diff(results, log_paths, None) + + print(diff2) + print(diff1) diff --git a/selfdrive/test/process_replay/model_replay.py b/selfdrive/test/process_replay/model_replay.py index 0cdaf861e8..08f86b2b5e 100755 --- a/selfdrive/test/process_replay/model_replay.py +++ b/selfdrive/test/process_replay/model_replay.py @@ -10,8 +10,7 @@ from openpilot.common.params import Params from openpilot.system.hardware import PC from openpilot.selfdrive.manager.process_config import managed_processes from openpilot.selfdrive.test.openpilotci import BASE_URL, get_url -from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs -from openpilot.selfdrive.test.process_replay.test_processes import format_diff +from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs, format_diff from openpilot.selfdrive.test.process_replay.process_replay import get_process_config, replay_process from openpilot.system.version import get_commit from openpilot.tools.lib.framereader import FrameReader diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index 973c16971e..e36548d7e5 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -9,7 +9,7 @@ from typing import Any, DefaultDict, Dict from openpilot.selfdrive.car.car_helpers import interface_names from openpilot.selfdrive.test.openpilotci import get_url, upload_file -from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs +from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs, format_diff from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, PROC_REPLAY_DIR, FAKEDATA, check_openpilot_enabled, replay_process from openpilot.system.version import get_commit from openpilot.tools.lib.filereader import FileReader @@ -115,45 +115,6 @@ def test_process(cfg, lr, segment, ref_log_path, new_log_path, ignore_fields=Non return str(e), log_msgs -def format_diff(results, log_paths, ref_commit): - diff1, diff2 = "", "" - diff2 += f"***** tested against commit {ref_commit} *****\n" - - failed = False - for segment, result in list(results.items()): - diff1 += f"***** results for segment {segment} *****\n" - diff2 += f"***** differences for segment {segment} *****\n" - - for proc, diff in list(result.items()): - # long diff - diff2 += f"*** process: {proc} ***\n" - diff2 += f"\tref: {log_paths[segment][proc]['ref']}\n" - diff2 += f"\tnew: {log_paths[segment][proc]['new']}\n\n" - - # short diff - diff1 += f" {proc}\n" - if isinstance(diff, str): - diff1 += f" ref: {log_paths[segment][proc]['ref']}\n" - diff1 += f" new: {log_paths[segment][proc]['new']}\n\n" - diff1 += f" {diff}\n" - failed = True - elif len(diff): - diff1 += f" ref: {log_paths[segment][proc]['ref']}\n" - diff1 += f" new: {log_paths[segment][proc]['new']}\n\n" - - cnt: Dict[str, int] = {} - for d in diff: - diff2 += f"\t{str(d)}\n" - - k = str(d[1]) - cnt[k] = 1 if k not in cnt else cnt[k] + 1 - - for k, v in sorted(cnt.items()): - diff1 += f" {k}: {v}\n" - failed = True - return diff1, diff2, failed - - if __name__ == "__main__": all_cars = {car for car, _ in segments} all_procs = {cfg.proc_name for cfg in CONFIGS if cfg.proc_name not in EXCLUDED_PROCS}