sadmen
/
openpilot_comma
mirror of https://github.com/commaai/openpilot.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Process replay enhancements (#1153)

Browse Source 
* process replay: exclude processes and cars

old-commit-hash: 796d5c8cca
commatwo_master
Adeeb 5 years ago committed by GitHub
parent c69cdd01d5
commit c6f9b68ef9
4 changed files with 146 additions and 86 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      selfdrive/car/car_helpers.py
  2. 21
      selfdrive/test/process_replay/compare_logs.py
  3. 10
      selfdrive/test/process_replay/process_replay.py
  4. 198
      selfdrive/test/process_replay/test_processes.py

3
selfdrive/car/car_helpers.py
Unescape View File

@ -56,7 +56,8 @@ def _get_interface_names():
# imports from directory selfdrive/car/<name>/
interfaces = load_interfaces(_get_interface_names())
interface_names = _get_interface_names()
interfaces = load_interfaces(interface_names)
def only_toyota_left(candidate_cars):

21
selfdrive/test/process_replay/compare_logs.py
Unescape View File

@ -2,6 +2,7 @@
import bz2
import os
import sys
import numbers
import dictdiffer
if "CI" in os.environ:
@ -22,7 +23,7 @@ def save_log(dest, log_msgs):
def remove_ignored_fields(msg, ignore):
msg = msg.as_builder()
for key, val in ignore:
for key in ignore:
attr = msg
keys = key.split(".")
if msg.which() not in key and len(keys) > 1:
@ -34,21 +35,29 @@ def remove_ignored_fields(msg, ignore):
except:
break
else:
v = getattr(attr, keys[-1])
if isinstance(v, bool):
val = False
elif isinstance(v, numbers.Number):
val = 0
else:
raise NotImplementedError
setattr(attr, keys[-1], val)
return msg.as_reader()
def compare_logs(log1, log2, ignore=[]):
def compare_logs(log1, log2, ignore_fields=[], ignore_msgs=[]):
filter_msgs = lambda m: m.which() not in ignore_msgs
log1, log2 = [list(filter(filter_msgs, log)) for log in (log1, log2)]
assert len(log1) == len(log2), "logs are not same length: " + str(len(log1)) + " VS " + str(len(log2))
ignore_fields = [k for k, v in ignore]
diff = []
for msg1, msg2 in tqdm(zip(log1, log2)):
if msg1.which() != msg2.which():
print(msg1, msg2)
assert False, "msgs not aligned between logs"
raise Exception("msgs not aligned between logs")
msg1_bytes = remove_ignored_fields(msg1, ignore).as_builder().to_bytes()
msg2_bytes = remove_ignored_fields(msg2, ignore).as_builder().to_bytes()
msg1_bytes = remove_ignored_fields(msg1, ignore_fields).as_builder().to_bytes()
msg2_bytes = remove_ignored_fields(msg2, ignore_fields).as_builder().to_bytes()
if msg1_bytes != msg2_bytes:
msg1_dict = msg1.to_dict(verbose=True)

10
selfdrive/test/process_replay/process_replay.py
Unescape View File

@ -200,7 +200,7 @@ CONFIGS = [
"thermal": [], "health": [], "liveCalibration": [], "dMonitoringState": [], "plan": [], "pathPlan": [], "gpsLocation": [],
"model": [],
},
ignore=[("logMonoTime", 0), ("valid", True), ("controlsState.startMonoTime", 0), ("controlsState.cumLagMs", 0)],
ignore=["logMonoTime", "valid", "controlsState.startMonoTime", "controlsState.cumLagMs"],
init_callback=fingerprint,
should_recv_callback=None,
),
@ -210,7 +210,7 @@ CONFIGS = [
"can": ["radarState", "liveTracks"],
"liveParameters": [], "controlsState": [], "model": [],
},
ignore=[("logMonoTime", 0), ("valid", True), ("radarState.cumLagMs", 0)],
ignore=["logMonoTime", "valid", "radarState.cumLagMs"],
init_callback=get_car_params,
should_recv_callback=radar_rcv_callback,
),
@ -220,7 +220,7 @@ CONFIGS = [
"model": ["pathPlan"], "radarState": ["plan"],
"carState": [], "controlsState": [], "liveParameters": [],
},
ignore=[("logMonoTime", 0), ("valid", True), ("plan.processingDelay", 0)],
ignore=["logMonoTime", "valid", "plan.processingDelay"],
init_callback=get_car_params,
should_recv_callback=None,
),
@ -229,7 +229,7 @@ CONFIGS = [
pub_sub={
"cameraOdometry": ["liveCalibration"]
},
ignore=[("logMonoTime", 0), ("valid", True)],
ignore=["logMonoTime", "valid"],
init_callback=get_car_params,
should_recv_callback=calibration_rcv_callback,
),
@ -239,7 +239,7 @@ CONFIGS = [
"driverState": ["dMonitoringState"],
"liveCalibration": [], "carState": [], "model": [], "gpsLocation": [],
},
ignore=[("logMonoTime", 0), ("valid", True)],
ignore=["logMonoTime", "valid"],
init_callback=get_car_params,
should_recv_callback=None,
),

198
selfdrive/test/process_replay/test_processes.py
Unescape View File

@ -1,117 +1,167 @@
#!/usr/bin/env python3
import argparse
import os
import requests
import sys
import tempfile
from selfdrive.car.car_helpers import interface_names
from selfdrive.test.process_replay.compare_logs import compare_logs
from selfdrive.test.process_replay.process_replay import replay_process, CONFIGS
from tools.lib.logreader import LogReader
segments = [
"0375fdf7b1ce594d|2019-06-13--08-32-25--3", # HONDA.ACCORD
"99c94dc769b5d96e|2019-08-03--14-19-59--2", # HONDA.CIVIC
"cce908f7eb8db67d|2019-08-02--15-09-51--3", # TOYOTA.COROLLA_TSS2
"7ad88f53d406b787|2019-07-09--10-18-56--8", # GM.VOLT
"704b2230eb5190d6|2019-07-06--19-29-10--0", # HYUNDAI.KIA_SORENTO
"b6e1317e1bfbefa6|2019-07-06--04-05-26--5", # CHRYSLER.JEEP_CHEROKEE
"7873afaf022d36e2|2019-07-03--18-46-44--0", # SUBARU.IMPREZA
"b0c9d2329ad1606b|2020-02-19--16-29-36--7", # VW.GOLF
("HONDA", "0375fdf7b1ce594d|2019-06-13--08-32-25--3"), # HONDA.ACCORD
("HONDA", "99c94dc769b5d96e|2019-08-03--14-19-59--2"), # HONDA.CIVIC
("TOYOTA", "cce908f7eb8db67d|2019-08-02--15-09-51--3"), # TOYOTA.COROLLA_TSS2
("GM", "7ad88f53d406b787|2019-07-09--10-18-56--8"), # GM.VOLT
("HYUNDAI", "704b2230eb5190d6|2019-07-06--19-29-10--0"), # HYUNDAI.KIA_SORENTO
("CHRYSLER", "b6e1317e1bfbefa6|2019-07-06--04-05-26--5"), # CHRYSLER.JEEP_CHEROKEE
("SUBARU", "7873afaf022d36e2|2019-07-03--18-46-44--0"), # SUBARU.IMPREZA
("VOLKSWAGEN", "b0c9d2329ad1606b|2020-02-19--16-29-36--7"), # VW.GOLF
]
# ford doesn't need to be tested until a full port is done
excluded_interfaces = ["mock", "ford"]
BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/"
# run the full test (including checks) when no args given
FULL_TEST = len(sys.argv) <= 1
def get_segment(segment_name):
route_name, segment_num = segment_name.rsplit("--", 1)
rlog_url = "https://commadataci.blob.core.windows.net/openpilotci/%s/%s/rlog.bz2" \
% (route_name.replace("|", "/"), segment_num)
r = requests.get(rlog_url)
if r.status_code != 200:
return None
rlog_url = BASE_URL + "%s/%s/rlog.bz2" % (route_name.replace("|", "/"), segment_num)
req = requests.get(rlog_url)
assert req.status_code == 200, ("Failed to download log for %s" % segment_name)
with tempfile.NamedTemporaryFile(delete=False, suffix=".bz2") as f:
f.write(r.content)
f.write(req.content)
return f.name
def test_process(cfg, lr, cmp_log_fn, ignore_fields=[], ignore_msgs=[]):
if not os.path.isfile(cmp_log_fn):
req = requests.get(BASE_URL + os.path.basename(cmp_log_fn))
assert req.status_code == 200, ("Failed to download %s" % cmp_log_fn)
with tempfile.NamedTemporaryFile(suffix=".bz2") as f:
f.write(req.content)
f.flush()
f.seek(0)
cmp_log_msgs = list(LogReader(f.name))
else:
cmp_log_msgs = list(LogReader(cmp_log_fn))
log_msgs = replay_process(cfg, lr)
# check to make sure openpilot is engaged in the route
# TODO: update routes so enable check can run
# failed enable check: honda bosch, hyundai, chrysler, and subaru
if cfg.proc_name == "controlsd" and FULL_TEST and False:
for msg in log_msgs:
if msg.which() == "controlsState":
if msg.controlsState.active:
break
else:
segment = cmp_log_fn.split("/")[-1].split("_")[0]
raise Exception("Route never enabled: %s" % segment)
return compare_logs(cmp_log_msgs, log_msgs, ignore_fields+cfg.ignore, ignore_msgs)
def format_diff(results, ref_commit):
diff1, diff2 = "", ""
diff2 += "***** tested against commit %s *****\n" % ref_commit
failed = False
for segment, result in list(results.items()):
diff1 += "***** results for segment %s *****\n" % segment
diff2 += "***** differences for segment %s *****\n" % segment
for proc, diff in list(result.items()):
diff1 += "\t%s\n" % proc
diff2 += "*** process: %s ***\n" % proc
if isinstance(diff, str):
diff1 += "\t\t%s\n" % diff
failed = True
elif len(diff):
cnt = {}
for d in diff:
diff2 += "\t%s\n" % str(d)
k = str(d[1])
cnt[k] = 1 if k not in cnt else cnt[k] + 1
for k, v in sorted(cnt.items()):
diff1 += "\t\t%s: %s\n" % (k, v)
failed = True
return diff1, diff2, failed
if __name__ == "__main__":
process_replay_dir = os.path.dirname(os.path.abspath(__file__))
ref_commit_fn = os.path.join(process_replay_dir, "ref_commit")
parser = argparse.ArgumentParser(description="Regression test to identify changes in a process's output")
# whitelist has precedence over blacklist in case both are defined
parser.add_argument("--whitelist-procs", type=str, nargs="*", default=[],
help="Whitelist given processes from the test (e.g. controlsd)")
parser.add_argument("--whitelist-cars", type=str, nargs="*", default=[],
help="Whitelist given cars from the test (e.g. HONDA)")
parser.add_argument("--blacklist-procs", type=str, nargs="*", default=[],
help="Blacklist given processes from the test (e.g. controlsd)")
parser.add_argument("--blacklist-cars", type=str, nargs="*", default=[],
help="Blacklist given cars from the test (e.g. HONDA)")
parser.add_argument("--ignore-fields", type=str, nargs="*", default=[],
help="Extra fields or msgs to ignore (e.g. carState.events)")
parser.add_argument("--ignore-msgs", type=str, nargs="*", default=[],
help="Msgs to ignore (e.g. carEvents)")
args = parser.parse_args()
cars_whitelisted = len(args.whitelist_cars) > 0
procs_whitelisted = len(args.whitelist_procs) > 0
if not os.path.isfile(ref_commit_fn):
process_replay_dir = os.path.dirname(os.path.abspath(__file__))
try:
ref_commit = open(os.path.join(process_replay_dir, "ref_commit")).read().strip()
except:
print("couldn't find reference commit")
sys.exit(1)
ref_commit = open(ref_commit_fn).read().strip()
print("***** testing against commit %s *****" % ref_commit)
# check to make sure all car brands are tested
if FULL_TEST:
tested_cars = set(c.lower() for c, _ in segments)
untested = (set(interface_names) - set(excluded_interfaces)) - tested_cars
assert len(untested) == 0, "Cars missing routes: %s" % (str(untested))
results = {}
for segment in segments:
for car_brand, segment in segments:
if (cars_whitelisted and car_brand.upper() not in args.whitelist_cars) or \
(not cars_whitelisted and car_brand.upper() in args.blacklist_cars):
continue
print("***** testing route segment %s *****\n" % segment)
results[segment] = {}
rlog_fn = get_segment(segment)
if rlog_fn is None:
print("failed to get segment %s" % segment)
sys.exit(1)
lr = LogReader(rlog_fn)
for cfg in CONFIGS:
log_msgs = replay_process(cfg, lr)
log_fn = os.path.join(process_replay_dir, "%s_%s_%s.bz2" % (segment, cfg.proc_name, ref_commit))
if not os.path.isfile(log_fn):
url = "https://commadataci.blob.core.windows.net/openpilotci/"
req = requests.get(url + os.path.basename(log_fn))
if req.status_code != 200:
results[segment][cfg.proc_name] = "failed to download comparison log"
continue
with tempfile.NamedTemporaryFile(suffix=".bz2") as f:
f.write(req.content)
f.flush()
f.seek(0)
cmp_log_msgs = list(LogReader(f.name))
else:
cmp_log_msgs = list(LogReader(log_fn))
diff = compare_logs(cmp_log_msgs, log_msgs, cfg.ignore)
results[segment][cfg.proc_name] = diff
if (procs_whitelisted and cfg.proc_name not in args.whitelist_procs) or \
(not procs_whitelisted and cfg.proc_name in args.blacklist_procs):
continue
cmp_log_fn = os.path.join(process_replay_dir, "%s_%s_%s.bz2" % (segment, cfg.proc_name, ref_commit))
results[segment][cfg.proc_name] = test_process(cfg, lr, cmp_log_fn, args.ignore_fields, args.ignore_msgs)
os.remove(rlog_fn)
failed = False
diff1, diff2, failed = format_diff(results, ref_commit)
with open(os.path.join(process_replay_dir, "diff.txt"), "w") as f:
f.write("***** tested against commit %s *****\n" % ref_commit)
for segment, result in list(results.items()):
f.write("***** differences for segment %s *****\n" % segment)
print("***** results for segment %s *****" % segment)
f.write(diff2)
print(diff1)
for proc, diff in list(result.items()):
f.write("*** process: %s ***\n" % proc)
print("\t%s" % proc)
if isinstance(diff, str):
print("\t\t%s" % diff)
failed = True
elif len(diff):
cnt = {}
for d in diff:
f.write("\t%s\n" % str(d))
k = str(d[1])
cnt[k] = 1 if k not in cnt else cnt[k] + 1
for k, v in sorted(cnt.items()):
print("\t\t%s: %s" % (k, v))
failed = True
if failed:
print("TEST FAILED")
else:
print("TEST SUCCEEDED")
print("TEST", "FAILED" if failed else "SUCCEEDED")
print("\n\nTo update the reference logs for this test run:")
print("./update_refs.py")

Write Preview
Loading…
Cancel
Save