diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py index ed82ba688d..ff00e9b909 100644 --- a/selfdrive/car/car_helpers.py +++ b/selfdrive/car/car_helpers.py @@ -56,7 +56,8 @@ def _get_interface_names(): # imports from directory selfdrive/car// -interfaces = load_interfaces(_get_interface_names()) +interface_names = _get_interface_names() +interfaces = load_interfaces(interface_names) def only_toyota_left(candidate_cars): diff --git a/selfdrive/test/process_replay/compare_logs.py b/selfdrive/test/process_replay/compare_logs.py index 82e701a75f..91eefafd27 100755 --- a/selfdrive/test/process_replay/compare_logs.py +++ b/selfdrive/test/process_replay/compare_logs.py @@ -2,6 +2,7 @@ import bz2 import os import sys +import numbers import dictdiffer if "CI" in os.environ: @@ -22,7 +23,7 @@ def save_log(dest, log_msgs): def remove_ignored_fields(msg, ignore): msg = msg.as_builder() - for key, val in ignore: + for key in ignore: attr = msg keys = key.split(".") if msg.which() not in key and len(keys) > 1: @@ -34,21 +35,29 @@ def remove_ignored_fields(msg, ignore): except: break else: + v = getattr(attr, keys[-1]) + if isinstance(v, bool): + val = False + elif isinstance(v, numbers.Number): + val = 0 + else: + raise NotImplementedError setattr(attr, keys[-1], val) return msg.as_reader() -def compare_logs(log1, log2, ignore=[]): +def compare_logs(log1, log2, ignore_fields=[], ignore_msgs=[]): + filter_msgs = lambda m: m.which() not in ignore_msgs + log1, log2 = [list(filter(filter_msgs, log)) for log in (log1, log2)] assert len(log1) == len(log2), "logs are not same length: " + str(len(log1)) + " VS " + str(len(log2)) - ignore_fields = [k for k, v in ignore] diff = [] for msg1, msg2 in tqdm(zip(log1, log2)): if msg1.which() != msg2.which(): print(msg1, msg2) - assert False, "msgs not aligned between logs" + raise Exception("msgs not aligned between logs") - msg1_bytes = remove_ignored_fields(msg1, ignore).as_builder().to_bytes() - msg2_bytes = remove_ignored_fields(msg2, ignore).as_builder().to_bytes() + msg1_bytes = remove_ignored_fields(msg1, ignore_fields).as_builder().to_bytes() + msg2_bytes = remove_ignored_fields(msg2, ignore_fields).as_builder().to_bytes() if msg1_bytes != msg2_bytes: msg1_dict = msg1.to_dict(verbose=True) diff --git a/selfdrive/test/process_replay/process_replay.py b/selfdrive/test/process_replay/process_replay.py index 9b27308b20..129bb85952 100755 --- a/selfdrive/test/process_replay/process_replay.py +++ b/selfdrive/test/process_replay/process_replay.py @@ -200,7 +200,7 @@ CONFIGS = [ "thermal": [], "health": [], "liveCalibration": [], "dMonitoringState": [], "plan": [], "pathPlan": [], "gpsLocation": [], "model": [], }, - ignore=[("logMonoTime", 0), ("valid", True), ("controlsState.startMonoTime", 0), ("controlsState.cumLagMs", 0)], + ignore=["logMonoTime", "valid", "controlsState.startMonoTime", "controlsState.cumLagMs"], init_callback=fingerprint, should_recv_callback=None, ), @@ -210,7 +210,7 @@ CONFIGS = [ "can": ["radarState", "liveTracks"], "liveParameters": [], "controlsState": [], "model": [], }, - ignore=[("logMonoTime", 0), ("valid", True), ("radarState.cumLagMs", 0)], + ignore=["logMonoTime", "valid", "radarState.cumLagMs"], init_callback=get_car_params, should_recv_callback=radar_rcv_callback, ), @@ -220,7 +220,7 @@ CONFIGS = [ "model": ["pathPlan"], "radarState": ["plan"], "carState": [], "controlsState": [], "liveParameters": [], }, - ignore=[("logMonoTime", 0), ("valid", True), ("plan.processingDelay", 0)], + ignore=["logMonoTime", "valid", "plan.processingDelay"], init_callback=get_car_params, should_recv_callback=None, ), @@ -229,7 +229,7 @@ CONFIGS = [ pub_sub={ "cameraOdometry": ["liveCalibration"] }, - ignore=[("logMonoTime", 0), ("valid", True)], + ignore=["logMonoTime", "valid"], init_callback=get_car_params, should_recv_callback=calibration_rcv_callback, ), @@ -239,7 +239,7 @@ CONFIGS = [ "driverState": ["dMonitoringState"], "liveCalibration": [], "carState": [], "model": [], "gpsLocation": [], }, - ignore=[("logMonoTime", 0), ("valid", True)], + ignore=["logMonoTime", "valid"], init_callback=get_car_params, should_recv_callback=None, ), diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index c5f52b8c9b..94d831ac03 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -1,117 +1,167 @@ #!/usr/bin/env python3 +import argparse import os import requests import sys import tempfile +from selfdrive.car.car_helpers import interface_names from selfdrive.test.process_replay.compare_logs import compare_logs from selfdrive.test.process_replay.process_replay import replay_process, CONFIGS from tools.lib.logreader import LogReader segments = [ - "0375fdf7b1ce594d|2019-06-13--08-32-25--3", # HONDA.ACCORD - "99c94dc769b5d96e|2019-08-03--14-19-59--2", # HONDA.CIVIC - "cce908f7eb8db67d|2019-08-02--15-09-51--3", # TOYOTA.COROLLA_TSS2 - "7ad88f53d406b787|2019-07-09--10-18-56--8", # GM.VOLT - "704b2230eb5190d6|2019-07-06--19-29-10--0", # HYUNDAI.KIA_SORENTO - "b6e1317e1bfbefa6|2019-07-06--04-05-26--5", # CHRYSLER.JEEP_CHEROKEE - "7873afaf022d36e2|2019-07-03--18-46-44--0", # SUBARU.IMPREZA - "b0c9d2329ad1606b|2020-02-19--16-29-36--7", # VW.GOLF + ("HONDA", "0375fdf7b1ce594d|2019-06-13--08-32-25--3"), # HONDA.ACCORD + ("HONDA", "99c94dc769b5d96e|2019-08-03--14-19-59--2"), # HONDA.CIVIC + ("TOYOTA", "cce908f7eb8db67d|2019-08-02--15-09-51--3"), # TOYOTA.COROLLA_TSS2 + ("GM", "7ad88f53d406b787|2019-07-09--10-18-56--8"), # GM.VOLT + ("HYUNDAI", "704b2230eb5190d6|2019-07-06--19-29-10--0"), # HYUNDAI.KIA_SORENTO + ("CHRYSLER", "b6e1317e1bfbefa6|2019-07-06--04-05-26--5"), # CHRYSLER.JEEP_CHEROKEE + ("SUBARU", "7873afaf022d36e2|2019-07-03--18-46-44--0"), # SUBARU.IMPREZA + ("VOLKSWAGEN", "b0c9d2329ad1606b|2020-02-19--16-29-36--7"), # VW.GOLF ] +# ford doesn't need to be tested until a full port is done +excluded_interfaces = ["mock", "ford"] + +BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/" + +# run the full test (including checks) when no args given +FULL_TEST = len(sys.argv) <= 1 + def get_segment(segment_name): route_name, segment_num = segment_name.rsplit("--", 1) - rlog_url = "https://commadataci.blob.core.windows.net/openpilotci/%s/%s/rlog.bz2" \ - % (route_name.replace("|", "/"), segment_num) - r = requests.get(rlog_url) - if r.status_code != 200: - return None + rlog_url = BASE_URL + "%s/%s/rlog.bz2" % (route_name.replace("|", "/"), segment_num) + req = requests.get(rlog_url) + assert req.status_code == 200, ("Failed to download log for %s" % segment_name) with tempfile.NamedTemporaryFile(delete=False, suffix=".bz2") as f: - f.write(r.content) + f.write(req.content) return f.name +def test_process(cfg, lr, cmp_log_fn, ignore_fields=[], ignore_msgs=[]): + if not os.path.isfile(cmp_log_fn): + req = requests.get(BASE_URL + os.path.basename(cmp_log_fn)) + assert req.status_code == 200, ("Failed to download %s" % cmp_log_fn) + + with tempfile.NamedTemporaryFile(suffix=".bz2") as f: + f.write(req.content) + f.flush() + f.seek(0) + cmp_log_msgs = list(LogReader(f.name)) + else: + cmp_log_msgs = list(LogReader(cmp_log_fn)) + + log_msgs = replay_process(cfg, lr) + + # check to make sure openpilot is engaged in the route + # TODO: update routes so enable check can run + # failed enable check: honda bosch, hyundai, chrysler, and subaru + if cfg.proc_name == "controlsd" and FULL_TEST and False: + for msg in log_msgs: + if msg.which() == "controlsState": + if msg.controlsState.active: + break + else: + segment = cmp_log_fn.split("/")[-1].split("_")[0] + raise Exception("Route never enabled: %s" % segment) + + return compare_logs(cmp_log_msgs, log_msgs, ignore_fields+cfg.ignore, ignore_msgs) + +def format_diff(results, ref_commit): + diff1, diff2 = "", "" + diff2 += "***** tested against commit %s *****\n" % ref_commit + + failed = False + for segment, result in list(results.items()): + diff1 += "***** results for segment %s *****\n" % segment + diff2 += "***** differences for segment %s *****\n" % segment + + for proc, diff in list(result.items()): + diff1 += "\t%s\n" % proc + diff2 += "*** process: %s ***\n" % proc + + if isinstance(diff, str): + diff1 += "\t\t%s\n" % diff + failed = True + elif len(diff): + cnt = {} + for d in diff: + diff2 += "\t%s\n" % str(d) + + k = str(d[1]) + cnt[k] = 1 if k not in cnt else cnt[k] + 1 + + for k, v in sorted(cnt.items()): + diff1 += "\t\t%s: %s\n" % (k, v) + failed = True + return diff1, diff2, failed + if __name__ == "__main__": - process_replay_dir = os.path.dirname(os.path.abspath(__file__)) - ref_commit_fn = os.path.join(process_replay_dir, "ref_commit") + parser = argparse.ArgumentParser(description="Regression test to identify changes in a process's output") + + # whitelist has precedence over blacklist in case both are defined + parser.add_argument("--whitelist-procs", type=str, nargs="*", default=[], + help="Whitelist given processes from the test (e.g. controlsd)") + parser.add_argument("--whitelist-cars", type=str, nargs="*", default=[], + help="Whitelist given cars from the test (e.g. HONDA)") + parser.add_argument("--blacklist-procs", type=str, nargs="*", default=[], + help="Blacklist given processes from the test (e.g. controlsd)") + parser.add_argument("--blacklist-cars", type=str, nargs="*", default=[], + help="Blacklist given cars from the test (e.g. HONDA)") + parser.add_argument("--ignore-fields", type=str, nargs="*", default=[], + help="Extra fields or msgs to ignore (e.g. carState.events)") + parser.add_argument("--ignore-msgs", type=str, nargs="*", default=[], + help="Msgs to ignore (e.g. carEvents)") + args = parser.parse_args() + + cars_whitelisted = len(args.whitelist_cars) > 0 + procs_whitelisted = len(args.whitelist_procs) > 0 - if not os.path.isfile(ref_commit_fn): + process_replay_dir = os.path.dirname(os.path.abspath(__file__)) + try: + ref_commit = open(os.path.join(process_replay_dir, "ref_commit")).read().strip() + except: print("couldn't find reference commit") sys.exit(1) - ref_commit = open(ref_commit_fn).read().strip() print("***** testing against commit %s *****" % ref_commit) + # check to make sure all car brands are tested + if FULL_TEST: + tested_cars = set(c.lower() for c, _ in segments) + untested = (set(interface_names) - set(excluded_interfaces)) - tested_cars + assert len(untested) == 0, "Cars missing routes: %s" % (str(untested)) + results = {} - for segment in segments: + for car_brand, segment in segments: + if (cars_whitelisted and car_brand.upper() not in args.whitelist_cars) or \ + (not cars_whitelisted and car_brand.upper() in args.blacklist_cars): + continue + print("***** testing route segment %s *****\n" % segment) results[segment] = {} rlog_fn = get_segment(segment) - - if rlog_fn is None: - print("failed to get segment %s" % segment) - sys.exit(1) - lr = LogReader(rlog_fn) for cfg in CONFIGS: - log_msgs = replay_process(cfg, lr) - - log_fn = os.path.join(process_replay_dir, "%s_%s_%s.bz2" % (segment, cfg.proc_name, ref_commit)) - - if not os.path.isfile(log_fn): - url = "https://commadataci.blob.core.windows.net/openpilotci/" - req = requests.get(url + os.path.basename(log_fn)) - if req.status_code != 200: - results[segment][cfg.proc_name] = "failed to download comparison log" - continue - - with tempfile.NamedTemporaryFile(suffix=".bz2") as f: - f.write(req.content) - f.flush() - f.seek(0) - cmp_log_msgs = list(LogReader(f.name)) - else: - cmp_log_msgs = list(LogReader(log_fn)) - - diff = compare_logs(cmp_log_msgs, log_msgs, cfg.ignore) - results[segment][cfg.proc_name] = diff + if (procs_whitelisted and cfg.proc_name not in args.whitelist_procs) or \ + (not procs_whitelisted and cfg.proc_name in args.blacklist_procs): + continue + + cmp_log_fn = os.path.join(process_replay_dir, "%s_%s_%s.bz2" % (segment, cfg.proc_name, ref_commit)) + results[segment][cfg.proc_name] = test_process(cfg, lr, cmp_log_fn, args.ignore_fields, args.ignore_msgs) os.remove(rlog_fn) - failed = False + diff1, diff2, failed = format_diff(results, ref_commit) with open(os.path.join(process_replay_dir, "diff.txt"), "w") as f: - f.write("***** tested against commit %s *****\n" % ref_commit) - - for segment, result in list(results.items()): - f.write("***** differences for segment %s *****\n" % segment) - print("***** results for segment %s *****" % segment) + f.write(diff2) + print(diff1) - for proc, diff in list(result.items()): - f.write("*** process: %s ***\n" % proc) - print("\t%s" % proc) - - if isinstance(diff, str): - print("\t\t%s" % diff) - failed = True - elif len(diff): - cnt = {} - for d in diff: - f.write("\t%s\n" % str(d)) - - k = str(d[1]) - cnt[k] = 1 if k not in cnt else cnt[k] + 1 - - for k, v in sorted(cnt.items()): - print("\t\t%s: %s" % (k, v)) - failed = True - - if failed: - print("TEST FAILED") - else: - print("TEST SUCCEEDED") + print("TEST", "FAILED" if failed else "SUCCEEDED") print("\n\nTo update the reference logs for this test run:") print("./update_refs.py")