process replay: only download used logs (#24661)

* only download wanted logs

* cleanup

* cache as an option

* cleanup

* Readme

* Revert "cache as an option"

This reverts commit 060ed4ade5.

* cleanup

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

selfdrive/test/process_replay/README.md

@@ -5,6 +5,7 @@ Process replay is a regression test designed to identify any changes in the outp
 If the test fails, make sure that you didn't unintentionally change anything. If there are intentional changes, the reference logs will be updated.
 
 Use `test_processes.py` to run the test locally.
+Use `FILEREADER_CACHE='1' test_processes.py` to cache log files. 
 
 Currently the following processes are tested:
 

selfdrive/test/process_replay/test_processes.py

@@ -131,12 +131,13 @@ def format_diff(results, ref_commit):
 
 
 if __name__ == "__main__":
-  parser = argparse.ArgumentParser(description="Regression test to identify changes in a process's output")
+  all_cars = {car for car, _ in segments}
+  all_procs = {cfg.proc_name for cfg in CONFIGS}
 
-  # whitelist has precedence over blacklist in case both are defined
+  parser = argparse.ArgumentParser(description="Regression test to identify changes in a process's output")
-  parser.add_argument("--whitelist-procs", type=str, nargs="*", default=[],
+  parser.add_argument("--whitelist-procs", type=str, nargs="*", default=all_procs,
                       help="Whitelist given processes from the test (e.g. controlsd)")
-  parser.add_argument("--whitelist-cars", type=str, nargs="*", default=[],
+  parser.add_argument("--whitelist-cars", type=str, nargs="*", default=all_cars,
                       help="Whitelist given cars from the test (e.g. HONDA)")
   parser.add_argument("--blacklist-procs", type=str, nargs="*", default=[],
                       help="Blacklist given processes from the test (e.g. controlsd)")
@@ -153,6 +154,10 @@ if __name__ == "__main__":
   parser.add_argument("-j", "--jobs", type=int, default=1)
   args = parser.parse_args()
 
+  tested_procs = set(args.whitelist_procs) - set(args.blacklist_procs)
+  tested_cars = set(args.whitelist_cars) - set(args.blacklist_cars)
+  tested_cars = {c.upper() for c in tested_cars}
+
   full_test = all(len(x) == 0 for x in (args.whitelist_procs, args.whitelist_cars, args.blacklist_procs, args.blacklist_cars, args.ignore_fields, args.ignore_msgs))
   upload = args.update_refs or args.upload_only
   os.makedirs(os.path.dirname(FAKEDATA), exist_ok=True)
@@ -180,20 +185,19 @@ if __name__ == "__main__":
 
   with concurrent.futures.ProcessPoolExecutor(max_workers=args.jobs) as pool:
     if not args.upload_only:
-      lreaders: Any = {}
+      download_segments = [seg for car, seg in segments if car in tested_cars]
-      p1 = pool.map(get_logreader, [seg for car, seg in segments])
+      lreaders: Dict[str, LogReader] = {}
-      for (segment, lr) in tqdm(p1, desc="Getting Logs", total=len(segments)):
+      p1 = pool.map(get_logreader, download_segments)
+      for segment, lr in tqdm(p1, desc="Getting Logs", total=len(download_segments)):
         lreaders[segment] = lr
 
     pool_args: Any = []
     for car_brand, segment in segments:
-      if (len(args.whitelist_cars) and car_brand.upper() not in args.whitelist_cars) or \
+      if car_brand not in tested_cars:
-         (not len(args.whitelist_cars) and car_brand.upper() in args.blacklist_cars):
         continue
 
       for cfg in CONFIGS:
-        if (len(args.whitelist_procs) and cfg.proc_name not in args.whitelist_procs) or \
+        if cfg.proc_name not in tested_procs:
-           (not len(args.whitelist_procs) and cfg.proc_name in args.blacklist_procs):
           continue
 
         cur_log_fn = os.path.join(FAKEDATA, f"{segment}_{cfg.proc_name}_{cur_commit}.bz2")