diff --git a/selfdrive/test/process_replay/README.md b/selfdrive/test/process_replay/README.md index 3add95f0a0..516646e922 100644 --- a/selfdrive/test/process_replay/README.md +++ b/selfdrive/test/process_replay/README.md @@ -1,4 +1,4 @@ -# process replay +# Process replay Process replay is a regression test designed to identify any changes in the output of a process. This test replays a segment through individual processes and compares the output to a known good replay. Each make is represented in the test with a segment. @@ -12,14 +12,34 @@ Currently the following processes are tested: * radard * plannerd * calibrationd +* dmonitoringd +* locationd +* paramsd * ubloxd +### Usage +``` +Usage: test_processes.py [-h] [--whitelist-procs PROCS] [--whitelist-cars CARS] [--blacklist-procs PROCS] + [--blacklist-cars CARS] [--ignore-fields FIELDS] [--ignore-msgs MSGS] [--timeout TIMEOUT] +Regression test to identify changes in a process's output +optional arguments: + -h, --help show this help message and exit + --whitelist-procs PROCS Whitelist given processes from the test (e.g. controlsd) + --whitelist-cars WHITELIST_CARS Whitelist given cars from the test (e.g. HONDA) + --blacklist-procs BLACKLIST_PROCS Blacklist given processes from the test (e.g. controlsd) + --blacklist-cars BLACKLIST_CARS Blacklist given cars from the test (e.g. HONDA) + --ignore-fields IGNORE_FIELDS Extra fields or msgs to ignore (e.g. carState.events) + --ignore-msgs IGNORE_MSGS Msgs to ignore (e.g. carEvents) + --update-refs Regenerates and uploads ref logs on current commit + --upload-only Skips testing processes and uploads logs from previous test run +``` + ## Forks -openpilot forks can use this test with their own reference logs +openpilot forks can use this test with their own reference logs, by default `test_proccess.py` saves logs locally. To generate new logs: -`./update_refs.py --no-upload` +`./test_processes.py` -Then, check in the new logs using git-lfs. Make sure to also include the updated `ref_commit` file. +Then, check in the new logs using git-lfs. Make sure to also update the `ref_commit` file to the current commit. diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index 61f24d62c0..75cc2c79f3 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -7,7 +7,7 @@ from typing import Any from selfdrive.car.car_helpers import interface_names from selfdrive.test.openpilotci import get_url, upload_file from selfdrive.test.process_replay.compare_logs import compare_logs, save_log -from selfdrive.test.process_replay.process_replay import CONFIGS, PROC_REPLAY_DIR, CI, check_enabled, replay_process +from selfdrive.test.process_replay.process_replay import CONFIGS, PROC_REPLAY_DIR, check_enabled, replay_process from selfdrive.version import get_commit from tools.lib.logreader import LogReader @@ -128,7 +128,7 @@ if __name__ == "__main__": parser.add_argument("--update-refs", action="store_true", help="Regenerates and uploads ref logs on current commit") parser.add_argument("--upload-only", action="store_true", - help="Uploads logs from previous test run") + help="Skips testing processes and uploads logs from previous test run") args = parser.parse_args() full_test = 0 == len(args.whitelist_procs) == len(args.whitelist_cars) == len(args.blacklist_procs) == \ diff --git a/selfdrive/test/process_replay/update_refs.py b/selfdrive/test/process_replay/update_refs.py deleted file mode 100755 index e7c38482c1..0000000000 --- a/selfdrive/test/process_replay/update_refs.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -import os -import sys - -from selfdrive.test.openpilotci import upload_file, get_url -from selfdrive.test.process_replay.compare_logs import save_log -from selfdrive.test.process_replay.process_replay import CONFIGS, PROC_REPLAY_DIR, CI, replay_process -from selfdrive.test.process_replay.test_processes import segments -from selfdrive.version import get_commit -from tools.lib.logreader import LogReader - - -if __name__ == "__main__": - no_upload = "--no-upload" in sys.argv - ref_commit_fn = os.path.join(PROC_REPLAY_DIR, "ref_commit") - - ref_commit = get_commit() - if ref_commit is None: - raise Exception("Couldn't get ref commit") - with open(ref_commit_fn, "w") as f: - f.write(ref_commit) - - # update refs/upload refs if CI - for car_brand, segment in segments: - if not CI: - r, n = segment.rsplit("--", 1) - lr = LogReader(get_url(r, n)) - - for cfg in CONFIGS: - log_fn = os.path.join(PROC_REPLAY_DIR, f"{segment}_{cfg.proc_name}_{ref_commit}.bz2") - if not CI: - log_msgs = replay_process(cfg, lr) - save_log(log_fn, log_msgs) - - if not os.path.exists(log_fn): - raise Exception(f"Cannot find log to upload: {log_fn}") - - if not no_upload: - print(f'Uploading: {log_fn}') - upload_file(log_fn, os.path.basename(log_fn)) - os.remove(log_fn) - - print(f'Done\nNew reference commit: {ref_commit}')