diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 752a5ec357..feb7cfa1d5 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -8,6 +8,7 @@ on: env: BASE_IMAGE: openpilot-base DOCKER_REGISTRY: ghcr.io/commaai + AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }} DOCKER_LOGIN: docker login ghcr.io -u adeebshihadeh -p ${{ secrets.CONTAINER_TOKEN }} BUILD: | @@ -316,6 +317,11 @@ jobs: ${{ env.RUN }} "scons -j$(nproc) && \ FILEREADER_CACHE=1 CI=1 coverage run selfdrive/test/process_replay/test_processes.py && \ coverage xml" + - name: Upload reference logs + if: ${{ failure() && github.event_name == 'pull_request' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }} + run: | + ${{ env.RUN }} "scons -j$(nproc) && \ + CI=1 AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py --upload-only" - name: "Upload coverage to Codecov" uses: codecov/codecov-action@v2 - name: Print diff diff --git a/selfdrive/test/openpilotci.py b/selfdrive/test/openpilotci.py index 6483b6717f..1d1b6ec423 100755 --- a/selfdrive/test/openpilotci.py +++ b/selfdrive/test/openpilotci.py @@ -4,17 +4,18 @@ import sys import subprocess BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/" - TOKEN_PATH = "/data/azure_token" + def get_url(route_name, segment_num, log_type="rlog"): ext = "hevc" if log_type.endswith('camera') else "bz2" return BASE_URL + f"{route_name.replace('|', '/')}/{segment_num}/{log_type}.{ext}" + def upload_file(path, name): from azure.storage.blob import BlockBlobService # pylint: disable=import-error - sas_token = None + sas_token = os.environ.get("AZURE_TOKEN", None) if os.path.isfile(TOKEN_PATH): sas_token = open(TOKEN_PATH).read().strip() @@ -25,6 +26,7 @@ def upload_file(path, name): service.create_blob_from_path("openpilotci", name, path) return "https://commadataci.blob.core.windows.net/openpilotci/" + name + if __name__ == "__main__": for f in sys.argv[1:]: name = os.path.basename(f) diff --git a/selfdrive/test/process_replay/README.md b/selfdrive/test/process_replay/README.md index 3add95f0a0..e760e4e135 100644 --- a/selfdrive/test/process_replay/README.md +++ b/selfdrive/test/process_replay/README.md @@ -1,4 +1,4 @@ -# process replay +# Process replay Process replay is a regression test designed to identify any changes in the output of a process. This test replays a segment through individual processes and compares the output to a known good replay. Each make is represented in the test with a segment. @@ -12,14 +12,34 @@ Currently the following processes are tested: * radard * plannerd * calibrationd +* dmonitoringd +* locationd +* paramsd * ubloxd +### Usage +``` +Usage: test_processes.py [-h] [--whitelist-procs PROCS] [--whitelist-cars CARS] [--blacklist-procs PROCS] + [--blacklist-cars CARS] [--ignore-fields FIELDS] [--ignore-msgs MSGS] [--update-refs] [--upload-only] +Regression test to identify changes in a process's output +optional arguments: + -h, --help show this help message and exit + --whitelist-procs PROCS Whitelist given processes from the test (e.g. controlsd) + --whitelist-cars WHITELIST_CARS Whitelist given cars from the test (e.g. HONDA) + --blacklist-procs BLACKLIST_PROCS Blacklist given processes from the test (e.g. controlsd) + --blacklist-cars BLACKLIST_CARS Blacklist given cars from the test (e.g. HONDA) + --ignore-fields IGNORE_FIELDS Extra fields or msgs to ignore (e.g. carState.events) + --ignore-msgs IGNORE_MSGS Msgs to ignore (e.g. carEvents) + --update-refs Updates reference logs using current commit + --upload-only Skips testing processes and uploads logs from previous test run +``` + ## Forks -openpilot forks can use this test with their own reference logs +openpilot forks can use this test with their own reference logs, by default `test_proccess.py` saves logs locally. To generate new logs: -`./update_refs.py --no-upload` +`./test_processes.py` -Then, check in the new logs using git-lfs. Make sure to also include the updated `ref_commit` file. +Then, check in the new logs using git-lfs. Make sure to also update the `ref_commit` file to the current commit. diff --git a/selfdrive/test/process_replay/process_replay.py b/selfdrive/test/process_replay/process_replay.py index e06d8198e4..c1b89713f3 100755 --- a/selfdrive/test/process_replay/process_replay.py +++ b/selfdrive/test/process_replay/process_replay.py @@ -24,6 +24,7 @@ from selfdrive.manager.process_config import managed_processes NUMPY_TOLERANCE = 1e-7 CI = "CI" in os.environ TIMEOUT = 15 +PROC_REPLAY_DIR = os.path.dirname(os.path.abspath(__file__)) ProcessConfig = namedtuple('ProcessConfig', ['proc_name', 'pub_sub', 'ignore', 'init_callback', 'should_recv_callback', 'tolerance', 'fake_pubsubmaster', 'submaster_config'], defaults=({},)) diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index 4f1f11eaf7..0a8fb9b454 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -5,9 +5,10 @@ import sys from typing import Any from selfdrive.car.car_helpers import interface_names -from selfdrive.test.openpilotci import get_url -from selfdrive.test.process_replay.compare_logs import compare_logs -from selfdrive.test.process_replay.process_replay import CONFIGS, replay_process, check_enabled +from selfdrive.test.openpilotci import get_url, upload_file +from selfdrive.test.process_replay.compare_logs import compare_logs, save_log +from selfdrive.test.process_replay.process_replay import CONFIGS, PROC_REPLAY_DIR, check_enabled, replay_process +from selfdrive.version import get_commit from tools.lib.logreader import LogReader @@ -50,32 +51,31 @@ segments = [ excluded_interfaces = ["mock", "ford", "mazda", "tesla"] BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/" +REF_COMMIT_FN = os.path.join(PROC_REPLAY_DIR, "ref_commit") -# run the full test (including checks) when no args given -FULL_TEST = len(sys.argv) <= 1 - -def test_process(cfg, lr, cmp_log_fn, ignore_fields=None, ignore_msgs=None): +def test_process(cfg, lr, ref_log_fn, ignore_fields=None, ignore_msgs=None): if ignore_fields is None: ignore_fields = [] if ignore_msgs is None: ignore_msgs = [] - cmp_log_path = cmp_log_fn if os.path.exists(cmp_log_fn) else BASE_URL + os.path.basename(cmp_log_fn) - cmp_log_msgs = list(LogReader(cmp_log_path)) + ref_log_path = ref_log_fn if os.path.exists(ref_log_fn) else BASE_URL + os.path.basename(ref_log_fn) + ref_log_msgs = list(LogReader(ref_log_path)) log_msgs = replay_process(cfg, lr) # check to make sure openpilot is engaged in the route if cfg.proc_name == "controlsd": if not check_enabled(log_msgs): - segment = cmp_log_fn.split("/")[-1].split("_")[0] + segment = ref_log_fn.split("/")[-1].split("_")[0] raise Exception(f"Route never enabled: {segment}") try: - return compare_logs(cmp_log_msgs, log_msgs, ignore_fields+cfg.ignore, ignore_msgs, cfg.tolerance) + return compare_logs(ref_log_msgs, log_msgs, ignore_fields + cfg.ignore, ignore_msgs, cfg.tolerance), log_msgs except Exception as e: - return str(e) + return str(e), log_msgs + def format_diff(results, ref_commit): diff1, diff2 = "", "" @@ -106,47 +106,57 @@ def format_diff(results, ref_commit): failed = True return diff1, diff2, failed -if __name__ == "__main__": +if __name__ == "__main__": parser = argparse.ArgumentParser(description="Regression test to identify changes in a process's output") # whitelist has precedence over blacklist in case both are defined parser.add_argument("--whitelist-procs", type=str, nargs="*", default=[], - help="Whitelist given processes from the test (e.g. controlsd)") + help="Whitelist given processes from the test (e.g. controlsd)") parser.add_argument("--whitelist-cars", type=str, nargs="*", default=[], - help="Whitelist given cars from the test (e.g. HONDA)") + help="Whitelist given cars from the test (e.g. HONDA)") parser.add_argument("--blacklist-procs", type=str, nargs="*", default=[], - help="Blacklist given processes from the test (e.g. controlsd)") + help="Blacklist given processes from the test (e.g. controlsd)") parser.add_argument("--blacklist-cars", type=str, nargs="*", default=[], - help="Blacklist given cars from the test (e.g. HONDA)") + help="Blacklist given cars from the test (e.g. HONDA)") parser.add_argument("--ignore-fields", type=str, nargs="*", default=[], - help="Extra fields or msgs to ignore (e.g. carState.events)") + help="Extra fields or msgs to ignore (e.g. carState.events)") parser.add_argument("--ignore-msgs", type=str, nargs="*", default=[], - help="Msgs to ignore (e.g. carEvents)") + help="Msgs to ignore (e.g. carEvents)") + parser.add_argument("--update-refs", action="store_true", + help="Updates reference logs using current commit") + parser.add_argument("--upload-only", action="store_true", + help="Skips testing processes and uploads logs from previous test run") args = parser.parse_args() - cars_whitelisted = len(args.whitelist_cars) > 0 - procs_whitelisted = len(args.whitelist_procs) > 0 + full_test = all(len(x) == 0 for x in (args.whitelist_procs, args.whitelist_cars, args.blacklist_procs, args.blacklist_cars, args.ignore_fields, args.ignore_msgs)) + upload = args.update_refs or args.upload_only + + if upload: + assert full_test, "Need to run full test when updating refs" - process_replay_dir = os.path.dirname(os.path.abspath(__file__)) try: - ref_commit = open(os.path.join(process_replay_dir, "ref_commit")).read().strip() + ref_commit = open(REF_COMMIT_FN).read().strip() except FileNotFoundError: - print("couldn't find reference commit") + print("Couldn't find reference commit") sys.exit(1) + cur_commit = get_commit() + if cur_commit is None: + raise Exception("Couldn't get current commit") + print(f"***** testing against commit {ref_commit} *****") # check to make sure all car brands are tested - if FULL_TEST: + if full_test: tested_cars = {c.lower() for c, _ in segments} untested = (set(interface_names) - set(excluded_interfaces)) - tested_cars assert len(untested) == 0, f"Cars missing routes: {str(untested)}" results: Any = {} for car_brand, segment in segments: - if (cars_whitelisted and car_brand.upper() not in args.whitelist_cars) or \ - (not cars_whitelisted and car_brand.upper() in args.blacklist_cars): + if (len(args.whitelist_cars) and car_brand.upper() not in args.whitelist_cars) or \ + (not len(args.whitelist_cars) and car_brand.upper() in args.blacklist_cars): continue print(f"***** testing route segment {segment} *****\n") @@ -157,23 +167,40 @@ if __name__ == "__main__": lr = LogReader(get_url(r, n)) for cfg in CONFIGS: - if (procs_whitelisted and cfg.proc_name not in args.whitelist_procs) or \ - (not procs_whitelisted and cfg.proc_name in args.blacklist_procs): + if (len(args.whitelist_procs) and cfg.proc_name not in args.whitelist_procs) or \ + (not len(args.whitelist_procs) and cfg.proc_name in args.blacklist_procs): continue - cmp_log_fn = os.path.join(process_replay_dir, f"{segment}_{cfg.proc_name}_{ref_commit}.bz2") - results[segment][cfg.proc_name] = test_process(cfg, lr, cmp_log_fn, args.ignore_fields, args.ignore_msgs) + cur_log_fn = os.path.join(PROC_REPLAY_DIR, f"{segment}_{cfg.proc_name}_{cur_commit}.bz2") + if not args.upload_only: + ref_log_fn = os.path.join(PROC_REPLAY_DIR, f"{segment}_{cfg.proc_name}_{ref_commit}.bz2") + results[segment][cfg.proc_name], log_msgs = test_process(cfg, lr, ref_log_fn, args.ignore_fields, args.ignore_msgs) + + # save logs so we can upload when updating refs + save_log(cur_log_fn, log_msgs) + + if upload: + print(f'Uploading: {os.path.basename(cur_log_fn)}') + assert os.path.exists(cur_log_fn), f"Cannot find log to upload: {cur_log_fn}" + upload_file(cur_log_fn, os.path.basename(cur_log_fn)) + os.remove(cur_log_fn) diff1, diff2, failed = format_diff(results, ref_commit) - with open(os.path.join(process_replay_dir, "diff.txt"), "w") as f: + with open(os.path.join(PROC_REPLAY_DIR, "diff.txt"), "w") as f: f.write(diff2) print(diff1) if failed: print("TEST FAILED") - print("\n\nTo update the reference logs for this test run:") - print("./update_refs.py") + if not upload: + print("\n\nTo push the new reference logs for this commit run:") + print("./test_processes.py --upload-only") else: print("TEST SUCCEEDED") + if upload: + with open(REF_COMMIT_FN, "w") as f: + f.write(cur_commit) + print(f"\n\nUpdated reference logs for commit: {cur_commit}") + sys.exit(int(failed)) diff --git a/selfdrive/test/process_replay/update_refs.py b/selfdrive/test/process_replay/update_refs.py deleted file mode 100755 index 0a3d95e714..0000000000 --- a/selfdrive/test/process_replay/update_refs.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python3 -import os -import sys - -from selfdrive.test.openpilotci import upload_file, get_url -from selfdrive.test.process_replay.compare_logs import save_log -from selfdrive.test.process_replay.process_replay import replay_process, CONFIGS -from selfdrive.test.process_replay.test_processes import segments -from selfdrive.version import get_commit -from tools.lib.logreader import LogReader - -if __name__ == "__main__": - - no_upload = "--no-upload" in sys.argv - - process_replay_dir = os.path.dirname(os.path.abspath(__file__)) - ref_commit_fn = os.path.join(process_replay_dir, "ref_commit") - - ref_commit = get_commit() - if ref_commit is None: - raise Exception("couldn't get ref commit") - with open(ref_commit_fn, "w") as f: - f.write(ref_commit) - - for car_brand, segment in segments: - r, n = segment.rsplit("--", 1) - lr = LogReader(get_url(r, n)) - - for cfg in CONFIGS: - log_msgs = replay_process(cfg, lr) - log_fn = os.path.join(process_replay_dir, f"{segment}_{cfg.proc_name}_{ref_commit}.bz2") - save_log(log_fn, log_msgs) - - if not no_upload: - upload_file(log_fn, os.path.basename(log_fn)) - os.remove(log_fn) - - print("done")