diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 752a5ec357..429a8184ef 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -314,8 +314,12 @@ jobs: - name: Run replay run: | ${{ env.RUN }} "scons -j$(nproc) && \ - FILEREADER_CACHE=1 CI=1 coverage run selfdrive/test/process_replay/test_processes.py && \ + FILEREADER_CACHE=1 CI=1 coverage run selfdrive/test/process_replay/test_processes.py --save-logs && \ coverage xml" + - name: Upload reference logs + if: ${{ failure() }} + run: | + echo "Uploading reference logs" - name: "Upload coverage to Codecov" uses: codecov/codecov-action@v2 - name: Print diff diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index 4f1f11eaf7..c44115d4c0 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -6,8 +6,8 @@ from typing import Any from selfdrive.car.car_helpers import interface_names from selfdrive.test.openpilotci import get_url -from selfdrive.test.process_replay.compare_logs import compare_logs -from selfdrive.test.process_replay.process_replay import CONFIGS, replay_process, check_enabled +from selfdrive.test.process_replay.compare_logs import compare_logs, save_log +from selfdrive.test.process_replay.process_replay import CONFIGS, check_enabled, replay_process from tools.lib.logreader import LogReader @@ -55,7 +55,7 @@ BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/" FULL_TEST = len(sys.argv) <= 1 -def test_process(cfg, lr, cmp_log_fn, ignore_fields=None, ignore_msgs=None): +def test_process(cfg, lr, cmp_log_fn, ignore_fields=None, ignore_msgs=None, save_logs=False): if ignore_fields is None: ignore_fields = [] if ignore_msgs is None: @@ -66,6 +66,10 @@ def test_process(cfg, lr, cmp_log_fn, ignore_fields=None, ignore_msgs=None): log_msgs = replay_process(cfg, lr) + # will overwrite any existing files, just like update_refs + if save_logs: + save_log(cmp_log_fn, log_msgs) + # check to make sure openpilot is engaged in the route if cfg.proc_name == "controlsd": if not check_enabled(log_msgs): @@ -77,6 +81,7 @@ def test_process(cfg, lr, cmp_log_fn, ignore_fields=None, ignore_msgs=None): except Exception as e: return str(e) + def format_diff(results, ref_commit): diff1, diff2 = "", "" diff2 += f"***** tested against commit {ref_commit} *****\n" @@ -106,23 +111,25 @@ def format_diff(results, ref_commit): failed = True return diff1, diff2, failed -if __name__ == "__main__": +if __name__ == "__main__": parser = argparse.ArgumentParser(description="Regression test to identify changes in a process's output") # whitelist has precedence over blacklist in case both are defined parser.add_argument("--whitelist-procs", type=str, nargs="*", default=[], - help="Whitelist given processes from the test (e.g. controlsd)") + help="Whitelist given processes from the test (e.g. controlsd)") parser.add_argument("--whitelist-cars", type=str, nargs="*", default=[], - help="Whitelist given cars from the test (e.g. HONDA)") + help="Whitelist given cars from the test (e.g. HONDA)") parser.add_argument("--blacklist-procs", type=str, nargs="*", default=[], - help="Blacklist given processes from the test (e.g. controlsd)") + help="Blacklist given processes from the test (e.g. controlsd)") parser.add_argument("--blacklist-cars", type=str, nargs="*", default=[], - help="Blacklist given cars from the test (e.g. HONDA)") + help="Blacklist given cars from the test (e.g. HONDA)") parser.add_argument("--ignore-fields", type=str, nargs="*", default=[], - help="Extra fields or msgs to ignore (e.g. carState.events)") + help="Extra fields or msgs to ignore (e.g. carState.events)") parser.add_argument("--ignore-msgs", type=str, nargs="*", default=[], - help="Msgs to ignore (e.g. carEvents)") + help="Msgs to ignore (e.g. carEvents)") + parser.add_argument("--save-logs", action="store_true", + help="Whether to save the generated log files") args = parser.parse_args() cars_whitelisted = len(args.whitelist_cars) > 0 @@ -136,6 +143,7 @@ if __name__ == "__main__": sys.exit(1) print(f"***** testing against commit {ref_commit} *****") + sys.exit(1) # check to make sure all car brands are tested if FULL_TEST: @@ -162,7 +170,7 @@ if __name__ == "__main__": continue cmp_log_fn = os.path.join(process_replay_dir, f"{segment}_{cfg.proc_name}_{ref_commit}.bz2") - results[segment][cfg.proc_name] = test_process(cfg, lr, cmp_log_fn, args.ignore_fields, args.ignore_msgs) + results[segment][cfg.proc_name] = test_process(cfg, lr, cmp_log_fn, args.ignore_fields, args.ignore_msgs, args.save_logs) diff1, diff2, failed = format_diff(results, ref_commit) with open(os.path.join(process_replay_dir, "diff.txt"), "w") as f: diff --git a/selfdrive/test/process_replay/update_refs.py b/selfdrive/test/process_replay/update_refs.py index 0a3d95e714..c683ba8391 100755 --- a/selfdrive/test/process_replay/update_refs.py +++ b/selfdrive/test/process_replay/update_refs.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 +import argparse import os -import sys from selfdrive.test.openpilotci import upload_file, get_url from selfdrive.test.process_replay.compare_logs import save_log @@ -10,8 +10,12 @@ from selfdrive.version import get_commit from tools.lib.logreader import LogReader if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Updates the reference logs for the current commit") - no_upload = "--no-upload" in sys.argv + parser.add_argument("--no-upload", action="store_true") + parser.add_argument("--only-upload", action="store_true") # TODO: split this out into own file upload_refs? + args = parser.parse_args() + assert args.no_upload != args.only_upload or not args.no_upload, "Both upload args can't be set" process_replay_dir = os.path.dirname(os.path.abspath(__file__)) ref_commit_fn = os.path.join(process_replay_dir, "ref_commit") @@ -23,6 +27,15 @@ if __name__ == "__main__": f.write(ref_commit) for car_brand, segment in segments: + if args.only_upload: + for cfg in CONFIGS: + log_fn = os.path.join(process_replay_dir, f"{segment}_{cfg.proc_name}_{ref_commit}.bz2") + if not os.path.exists(log_fn): + raise Exception("couldn't find file for uploading: {}".format(log_fn)) + upload_file(log_fn, os.path.basename(log_fn)) + os.remove(log_fn) + continue + r, n = segment.rsplit("--", 1) lr = LogReader(get_url(r, n)) @@ -31,7 +44,7 @@ if __name__ == "__main__": log_fn = os.path.join(process_replay_dir, f"{segment}_{cfg.proc_name}_{ref_commit}.bz2") save_log(log_fn, log_msgs) - if not no_upload: + if not args.no_upload: upload_file(log_fn, os.path.basename(log_fn)) os.remove(log_fn)