Process replay enhancements (#1153)

* process replay: exclude processes and cars

old-commit-hash: 796d5c8cca

selfdrive/car/car_helpers.py

@@ -56,7 +56,8 @@ def _get_interface_names():
 
 
 # imports from directory selfdrive/car/<name>/
-interfaces = load_interfaces(_get_interface_names())
+interface_names = _get_interface_names()
+interfaces = load_interfaces(interface_names)
 
 
 def only_toyota_left(candidate_cars):

selfdrive/test/process_replay/compare_logs.py

@@ -2,6 +2,7 @@
 import bz2
 import os
 import sys
+import numbers
 
 import dictdiffer
 if "CI" in os.environ:
@@ -22,7 +23,7 @@ def save_log(dest, log_msgs):
 
 def remove_ignored_fields(msg, ignore):
   msg = msg.as_builder()
-  for key, val in ignore:
+  for key in ignore:
     attr = msg
     keys = key.split(".")
     if msg.which() not in key and len(keys) > 1:
@@ -34,21 +35,29 @@ def remove_ignored_fields(msg, ignore):
       except:
         break
     else:
+      v = getattr(attr, keys[-1])
+      if isinstance(v, bool):
+        val = False
+      elif isinstance(v, numbers.Number):
+        val = 0
+      else:
+        raise NotImplementedError
       setattr(attr, keys[-1], val)
   return msg.as_reader()
 
-def compare_logs(log1, log2, ignore=[]):
+def compare_logs(log1, log2, ignore_fields=[], ignore_msgs=[]):
+  filter_msgs = lambda m: m.which() not in ignore_msgs
+  log1, log2 = [list(filter(filter_msgs, log)) for log in (log1, log2)]
   assert len(log1) == len(log2), "logs are not same length: " + str(len(log1)) + " VS " + str(len(log2))
 
-  ignore_fields = [k for k, v in ignore]
   diff = []
   for msg1, msg2 in tqdm(zip(log1, log2)):
     if msg1.which() != msg2.which():
       print(msg1, msg2)
-      assert False, "msgs not aligned between logs"
+      raise Exception("msgs not aligned between logs")
 
-    msg1_bytes = remove_ignored_fields(msg1, ignore).as_builder().to_bytes()
+    msg1_bytes = remove_ignored_fields(msg1, ignore_fields).as_builder().to_bytes()
-    msg2_bytes = remove_ignored_fields(msg2, ignore).as_builder().to_bytes()
+    msg2_bytes = remove_ignored_fields(msg2, ignore_fields).as_builder().to_bytes()
 
     if msg1_bytes != msg2_bytes:
       msg1_dict = msg1.to_dict(verbose=True)

selfdrive/test/process_replay/process_replay.py

@@ -200,7 +200,7 @@ CONFIGS = [
       "thermal": [], "health": [], "liveCalibration": [], "dMonitoringState": [], "plan": [], "pathPlan": [], "gpsLocation": [],
       "model": [],
     },
-    ignore=[("logMonoTime", 0), ("valid", True), ("controlsState.startMonoTime", 0), ("controlsState.cumLagMs", 0)],
+    ignore=["logMonoTime", "valid", "controlsState.startMonoTime", "controlsState.cumLagMs"],
     init_callback=fingerprint,
     should_recv_callback=None,
   ),
@@ -210,7 +210,7 @@ CONFIGS = [
       "can": ["radarState", "liveTracks"],
       "liveParameters":  [], "controlsState":  [], "model":  [],
     },
-    ignore=[("logMonoTime", 0), ("valid", True), ("radarState.cumLagMs", 0)],
+    ignore=["logMonoTime", "valid", "radarState.cumLagMs"],
     init_callback=get_car_params,
     should_recv_callback=radar_rcv_callback,
   ),
@@ -220,7 +220,7 @@ CONFIGS = [
       "model": ["pathPlan"], "radarState": ["plan"],
       "carState": [], "controlsState": [], "liveParameters": [],
     },
-    ignore=[("logMonoTime", 0), ("valid", True), ("plan.processingDelay", 0)],
+    ignore=["logMonoTime", "valid", "plan.processingDelay"],
     init_callback=get_car_params,
     should_recv_callback=None,
   ),
@@ -229,7 +229,7 @@ CONFIGS = [
     pub_sub={
       "cameraOdometry": ["liveCalibration"]
     },
-    ignore=[("logMonoTime", 0), ("valid", True)],
+    ignore=["logMonoTime", "valid"],
     init_callback=get_car_params,
     should_recv_callback=calibration_rcv_callback,
   ),
@@ -239,7 +239,7 @@ CONFIGS = [
       "driverState": ["dMonitoringState"],
       "liveCalibration": [], "carState": [], "model": [], "gpsLocation": [],
     },
-    ignore=[("logMonoTime", 0), ("valid", True)],
+    ignore=["logMonoTime", "valid"],
     init_callback=get_car_params,
     should_recv_callback=None,
   ),

selfdrive/test/process_replay/test_processes.py

@@ -1,73 +1,48 @@
 #!/usr/bin/env python3
+import argparse
 import os
 import requests
 import sys
 import tempfile
 
+from selfdrive.car.car_helpers import interface_names
 from selfdrive.test.process_replay.compare_logs import compare_logs
 from selfdrive.test.process_replay.process_replay import replay_process, CONFIGS
 from tools.lib.logreader import LogReader
 
 segments = [
-  "0375fdf7b1ce594d|2019-06-13--08-32-25--3", # HONDA.ACCORD
+  ("HONDA", "0375fdf7b1ce594d|2019-06-13--08-32-25--3"),      # HONDA.ACCORD
-  "99c94dc769b5d96e|2019-08-03--14-19-59--2", # HONDA.CIVIC
+  ("HONDA", "99c94dc769b5d96e|2019-08-03--14-19-59--2"),      # HONDA.CIVIC
-  "cce908f7eb8db67d|2019-08-02--15-09-51--3", # TOYOTA.COROLLA_TSS2
+  ("TOYOTA", "cce908f7eb8db67d|2019-08-02--15-09-51--3"),     # TOYOTA.COROLLA_TSS2
-  "7ad88f53d406b787|2019-07-09--10-18-56--8", # GM.VOLT
+  ("GM", "7ad88f53d406b787|2019-07-09--10-18-56--8"),         # GM.VOLT
-  "704b2230eb5190d6|2019-07-06--19-29-10--0", # HYUNDAI.KIA_SORENTO
+  ("HYUNDAI", "704b2230eb5190d6|2019-07-06--19-29-10--0"),    # HYUNDAI.KIA_SORENTO
-  "b6e1317e1bfbefa6|2019-07-06--04-05-26--5", # CHRYSLER.JEEP_CHEROKEE
+  ("CHRYSLER", "b6e1317e1bfbefa6|2019-07-06--04-05-26--5"),   # CHRYSLER.JEEP_CHEROKEE
-  "7873afaf022d36e2|2019-07-03--18-46-44--0", # SUBARU.IMPREZA
+  ("SUBARU", "7873afaf022d36e2|2019-07-03--18-46-44--0"),     # SUBARU.IMPREZA
-  "b0c9d2329ad1606b|2020-02-19--16-29-36--7", # VW.GOLF
+  ("VOLKSWAGEN", "b0c9d2329ad1606b|2020-02-19--16-29-36--7"), # VW.GOLF
 ]
 
+# ford doesn't need to be tested until a full port is done
+excluded_interfaces = ["mock", "ford"]
+
+BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/"
+
+# run the full test (including checks) when no args given
+FULL_TEST = len(sys.argv) <= 1
+
 def get_segment(segment_name):
   route_name, segment_num = segment_name.rsplit("--", 1)
-  rlog_url = "https://commadataci.blob.core.windows.net/openpilotci/%s/%s/rlog.bz2" \
+  rlog_url = BASE_URL + "%s/%s/rlog.bz2" % (route_name.replace("|", "/"), segment_num)
-                % (route_name.replace("|", "/"), segment_num)
+  req = requests.get(rlog_url)
-  r = requests.get(rlog_url)
+  assert req.status_code == 200, ("Failed to download log for %s" % segment_name)
-  if r.status_code != 200:
-    return None
 
   with tempfile.NamedTemporaryFile(delete=False, suffix=".bz2") as f:
-    f.write(r.content)
+    f.write(req.content)
     return f.name
 
-if __name__ == "__main__":
+def test_process(cfg, lr, cmp_log_fn, ignore_fields=[], ignore_msgs=[]):
-
+  if not os.path.isfile(cmp_log_fn):
-  process_replay_dir = os.path.dirname(os.path.abspath(__file__))
+    req = requests.get(BASE_URL + os.path.basename(cmp_log_fn))
-  ref_commit_fn = os.path.join(process_replay_dir, "ref_commit")
+    assert req.status_code == 200, ("Failed to download %s" % cmp_log_fn)
-
-  if not os.path.isfile(ref_commit_fn):
-    print("couldn't find reference commit")
-    sys.exit(1)
-
-  ref_commit = open(ref_commit_fn).read().strip()
-  print("***** testing against commit %s *****" % ref_commit)
-
-  results = {}
-  for segment in segments:
-    print("***** testing route segment %s *****\n" % segment)
-
-    results[segment] = {}
-
-    rlog_fn = get_segment(segment)
-
-    if rlog_fn is None:
-      print("failed to get segment %s" % segment)
-      sys.exit(1)
-
-    lr = LogReader(rlog_fn)
-
-    for cfg in CONFIGS:
-      log_msgs = replay_process(cfg, lr)
-
-      log_fn = os.path.join(process_replay_dir, "%s_%s_%s.bz2" % (segment, cfg.proc_name, ref_commit))
-
-      if not os.path.isfile(log_fn):
-        url = "https://commadataci.blob.core.windows.net/openpilotci/"
-        req = requests.get(url + os.path.basename(log_fn))
-        if req.status_code != 200:
-          results[segment][cfg.proc_name] = "failed to download comparison log"
-          continue
 
     with tempfile.NamedTemporaryFile(suffix=".bz2") as f:
       f.write(req.content)
@@ -75,43 +50,118 @@ if __name__ == "__main__":
       f.seek(0)
       cmp_log_msgs = list(LogReader(f.name))
   else:
-        cmp_log_msgs = list(LogReader(log_fn))
+    cmp_log_msgs = list(LogReader(cmp_log_fn))
 
-      diff = compare_logs(cmp_log_msgs, log_msgs, cfg.ignore)
+  log_msgs = replay_process(cfg, lr)
-      results[segment][cfg.proc_name] = diff
-    os.remove(rlog_fn)
 
-  failed = False
+  # check to make sure openpilot is engaged in the route
-  with open(os.path.join(process_replay_dir, "diff.txt"), "w") as f:
+  # TODO: update routes so enable check can run
-    f.write("***** tested against commit %s *****\n" % ref_commit)
+  #       failed enable check: honda bosch, hyundai, chrysler, and subaru
+  if cfg.proc_name == "controlsd" and FULL_TEST and False:
+    for msg in log_msgs:
+      if msg.which() == "controlsState":
+        if msg.controlsState.active:
+          break
+    else:
+      segment = cmp_log_fn.split("/")[-1].split("_")[0]
+      raise Exception("Route never enabled: %s" % segment)
+
+  return compare_logs(cmp_log_msgs, log_msgs, ignore_fields+cfg.ignore, ignore_msgs)
 
+def format_diff(results, ref_commit):
+  diff1, diff2 = "", ""
+  diff2 += "***** tested against commit %s *****\n" % ref_commit
+
+  failed = False
   for segment, result in list(results.items()):
-      f.write("***** differences for segment %s *****\n" % segment)
+    diff1 += "***** results for segment %s *****\n" % segment
-      print("***** results for segment %s *****" % segment)
+    diff2 += "***** differences for segment %s *****\n" % segment
 
     for proc, diff in list(result.items()):
-        f.write("*** process: %s ***\n" % proc)
+      diff1 += "\t%s\n" % proc
-        print("\t%s" % proc)
+      diff2 += "*** process: %s ***\n" % proc
 
       if isinstance(diff, str):
-          print("\t\t%s" % diff)
+        diff1 += "\t\t%s\n" % diff
         failed = True
       elif len(diff):
         cnt = {}
         for d in diff:
-            f.write("\t%s\n" % str(d))
+          diff2 += "\t%s\n" % str(d)
 
           k = str(d[1])
           cnt[k] = 1 if k not in cnt else cnt[k] + 1
 
         for k, v in sorted(cnt.items()):
-            print("\t\t%s: %s" % (k, v))
+          diff1 += "\t\t%s: %s\n" % (k, v)
         failed = True
+  return diff1, diff2, failed
 
-    if failed:
+if __name__ == "__main__":
-      print("TEST FAILED")
+
-    else:
+  parser = argparse.ArgumentParser(description="Regression test to identify changes in a process's output")
-      print("TEST SUCCEEDED")
+
+  # whitelist has precedence over blacklist in case both are defined
+  parser.add_argument("--whitelist-procs", type=str, nargs="*", default=[],
+                        help="Whitelist given processes from the test (e.g. controlsd)")
+  parser.add_argument("--whitelist-cars", type=str, nargs="*", default=[],
+                        help="Whitelist given cars from the test (e.g. HONDA)")
+  parser.add_argument("--blacklist-procs", type=str, nargs="*", default=[],
+                        help="Blacklist given processes from the test (e.g. controlsd)")
+  parser.add_argument("--blacklist-cars", type=str, nargs="*", default=[],
+                        help="Blacklist given cars from the test (e.g. HONDA)")
+  parser.add_argument("--ignore-fields", type=str, nargs="*", default=[],
+                        help="Extra fields or msgs to ignore (e.g. carState.events)")
+  parser.add_argument("--ignore-msgs", type=str, nargs="*", default=[],
+                        help="Msgs to ignore (e.g. carEvents)")
+  args = parser.parse_args()
+
+  cars_whitelisted = len(args.whitelist_cars) > 0
+  procs_whitelisted = len(args.whitelist_procs) > 0
+
+  process_replay_dir = os.path.dirname(os.path.abspath(__file__))
+  try:
+    ref_commit = open(os.path.join(process_replay_dir, "ref_commit")).read().strip()
+  except:
+    print("couldn't find reference commit")
+    sys.exit(1)
+
+  print("***** testing against commit %s *****" % ref_commit)
+
+  # check to make sure all car brands are tested
+  if FULL_TEST:
+    tested_cars = set(c.lower() for c, _ in segments)
+    untested = (set(interface_names) - set(excluded_interfaces)) - tested_cars
+    assert len(untested) == 0, "Cars missing routes: %s" % (str(untested))
+
+  results = {}
+  for car_brand, segment in segments:
+    if (cars_whitelisted and car_brand.upper() not in args.whitelist_cars) or \
+        (not cars_whitelisted and car_brand.upper() in args.blacklist_cars):
+      continue
+
+    print("***** testing route segment %s *****\n" % segment)
+
+    results[segment] = {}
+
+    rlog_fn = get_segment(segment)
+    lr = LogReader(rlog_fn)
+
+    for cfg in CONFIGS:
+      if (procs_whitelisted and cfg.proc_name not in args.whitelist_procs) or \
+          (not procs_whitelisted and cfg.proc_name in args.blacklist_procs):
+        continue
+
+      cmp_log_fn = os.path.join(process_replay_dir, "%s_%s_%s.bz2" % (segment, cfg.proc_name, ref_commit))
+      results[segment][cfg.proc_name] = test_process(cfg, lr, cmp_log_fn, args.ignore_fields, args.ignore_msgs)
+    os.remove(rlog_fn)
+
+  diff1, diff2, failed = format_diff(results, ref_commit)
+  with open(os.path.join(process_replay_dir, "diff.txt"), "w") as f:
+    f.write(diff2)
+  print(diff1)
+
+  print("TEST", "FAILED" if failed else "SUCCEEDED")
 
   print("\n\nTo update the reference logs for this test run:")
   print("./update_refs.py")