update README and delete update_refs

selfdrive/test/process_replay/README.md

@@ -1,4 +1,4 @@
-# process replay
+# Process replay
 
 Process replay is a regression test designed to identify any changes in the output of a process. This test replays a segment through individual processes and compares the output to a known good replay. Each make is represented in the test with a segment.
 
@@ -12,14 +12,34 @@ Currently the following processes are tested:
 * radard
 * plannerd
 * calibrationd
+* dmonitoringd
+* locationd
+* paramsd
 * ubloxd
 
+### Usage
+```
+Usage: test_processes.py [-h] [--whitelist-procs PROCS] [--whitelist-cars CARS] [--blacklist-procs PROCS]
+                         [--blacklist-cars CARS] [--ignore-fields FIELDS] [--ignore-msgs MSGS] [--timeout TIMEOUT]
+Regression test to identify changes in a process's output
+optional arguments:
+  -h, --help            show this help message and exit
+  --whitelist-procs PROCS               Whitelist given processes from the test (e.g. controlsd)
+  --whitelist-cars WHITELIST_CARS       Whitelist given cars from the test (e.g. HONDA)
+  --blacklist-procs BLACKLIST_PROCS     Blacklist given processes from the test (e.g. controlsd)
+  --blacklist-cars BLACKLIST_CARS       Blacklist given cars from the test (e.g. HONDA)
+  --ignore-fields IGNORE_FIELDS         Extra fields or msgs to ignore (e.g. carState.events)
+  --ignore-msgs IGNORE_MSGS             Msgs to ignore (e.g. carEvents)
+  --update-refs                         Regenerates and uploads ref logs on current commit
+  --upload-only                         Skips testing processes and uploads logs from previous test run
+```
+
 ## Forks
 
-openpilot forks can use this test with their own reference logs
+openpilot forks can use this test with their own reference logs, by default `test_proccess.py` saves logs locally.
 
 To generate new logs:
 
-`./update_refs.py --no-upload`
+`./test_processes.py`
 
-Then, check in the new logs using git-lfs. Make sure to also include the updated `ref_commit` file.
+Then, check in the new logs using git-lfs. Make sure to also update the `ref_commit` file to the current commit.

selfdrive/test/process_replay/test_processes.py

@@ -7,7 +7,7 @@ from typing import Any
 from selfdrive.car.car_helpers import interface_names
 from selfdrive.test.openpilotci import get_url, upload_file
 from selfdrive.test.process_replay.compare_logs import compare_logs, save_log
-from selfdrive.test.process_replay.process_replay import CONFIGS, PROC_REPLAY_DIR, CI, check_enabled, replay_process
+from selfdrive.test.process_replay.process_replay import CONFIGS, PROC_REPLAY_DIR, check_enabled, replay_process
 from selfdrive.version import get_commit
 from tools.lib.logreader import LogReader
 
@@ -128,7 +128,7 @@ if __name__ == "__main__":
   parser.add_argument("--update-refs", action="store_true",
                       help="Regenerates and uploads ref logs on current commit")
   parser.add_argument("--upload-only", action="store_true",
-                      help="Uploads logs from previous test run")
+                      help="Skips testing processes and uploads logs from previous test run")
   args = parser.parse_args()
 
   full_test = 0 == len(args.whitelist_procs) == len(args.whitelist_cars) == len(args.blacklist_procs) == \

selfdrive/test/process_replay/update_refs.py

@@ -1,43 +0,0 @@
-#!/usr/bin/env python3
-import os
-import sys
-
-from selfdrive.test.openpilotci import upload_file, get_url
-from selfdrive.test.process_replay.compare_logs import save_log
-from selfdrive.test.process_replay.process_replay import CONFIGS, PROC_REPLAY_DIR, CI, replay_process
-from selfdrive.test.process_replay.test_processes import segments
-from selfdrive.version import get_commit
-from tools.lib.logreader import LogReader
-
-
-if __name__ == "__main__":
-  no_upload = "--no-upload" in sys.argv
-  ref_commit_fn = os.path.join(PROC_REPLAY_DIR, "ref_commit")
-
-  ref_commit = get_commit()
-  if ref_commit is None:
-    raise Exception("Couldn't get ref commit")
-  with open(ref_commit_fn, "w") as f:
-    f.write(ref_commit)
-
-  # update refs/upload refs if CI
-  for car_brand, segment in segments:
-    if not CI:
-      r, n = segment.rsplit("--", 1)
-      lr = LogReader(get_url(r, n))
-
-    for cfg in CONFIGS:
-      log_fn = os.path.join(PROC_REPLAY_DIR, f"{segment}_{cfg.proc_name}_{ref_commit}.bz2")
-      if not CI:
-        log_msgs = replay_process(cfg, lr)
-        save_log(log_fn, log_msgs)
-
-      if not os.path.exists(log_fn):
-        raise Exception(f"Cannot find log to upload: {log_fn}")
-
-      if not no_upload:
-        print(f'Uploading: {log_fn}')
-        upload_file(log_fn, os.path.basename(log_fn))
-        os.remove(log_fn)
-
-  print(f'Done\nNew reference commit: {ref_commit}')