diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index d2174b409f..c5820a5f5a 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -228,6 +228,33 @@ jobs: - name: "Upload coverage to Codecov" uses: codecov/codecov-action@v3 + regen: + name: regen + runs-on: 'ubuntu-20.04' + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - uses: ./.github/workflows/setup-with-retry + - name: Cache test routes + id: dependency-cache + uses: actions/cache@v3 + with: + path: .ci_cache/comma_download_cache + key: regen-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/test_regen.py') }} + - name: Build base Docker image + run: eval "$BUILD" + - name: Build Docker image + run: eval "$BUILD_CL" + - name: Build openpilot + run: | + ${{ env.RUN }} "scons -j$(nproc)" + - name: Run regen + timeout-minutes: 30 + run: | + ${{ env.RUN_CL }} "ONNXCPU=1 $PYTEST -n auto --dist=loadscope selfdrive/test/process_replay/test_regen.py && \ + chmod -R 777 /tmp/comma_download_cache" + test_modeld: name: model tests runs-on: ubuntu-20.04 diff --git a/selfdrive/test/process_replay/regen_all.py b/selfdrive/test/process_replay/regen_all.py index df7c76a14d..b797b9b0da 100755 --- a/selfdrive/test/process_replay/regen_all.py +++ b/selfdrive/test/process_replay/regen_all.py @@ -29,15 +29,25 @@ def regen_job(segment, upload, disable_tqdm): if __name__ == "__main__": + all_cars = {car for car, _ in segments} + parser = argparse.ArgumentParser(description="Generate new segments from old ones") parser.add_argument("-j", "--jobs", type=int, default=1) parser.add_argument("--no-upload", action="store_true") + parser.add_argument("--whitelist-cars", type=str, nargs="*", default=all_cars, + help="Whitelist given cars from the test (e.g. HONDA)") + parser.add_argument("--blacklist-cars", type=str, nargs="*", default=[], + help="Blacklist given cars from the test (e.g. HONDA)") args = parser.parse_args() + tested_cars = set(args.whitelist_cars) - set(args.blacklist_cars) + tested_cars = {c.upper() for c in tested_cars} + tested_segments = [(car, segment) for car, segment in segments if car in tested_cars] + with concurrent.futures.ProcessPoolExecutor(max_workers=args.jobs) as pool: - p = pool.map(regen_job, segments, [not args.no_upload] * len(segments), [args.jobs > 1] * len(segments)) + p = pool.map(regen_job, tested_segments, [not args.no_upload] * len(tested_segments), [args.jobs > 1] * len(tested_segments)) msg = "Copy these new segments into test_processes.py:" - for seg in tqdm(p, desc="Generating segments", total=len(segments)): + for seg in tqdm(p, desc="Generating segments", total=len(tested_segments)): msg += "\n" + str(seg) print() print() diff --git a/selfdrive/test/process_replay/test_regen.py b/selfdrive/test/process_replay/test_regen.py new file mode 100755 index 0000000000..ec1277a76c --- /dev/null +++ b/selfdrive/test/process_replay/test_regen.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +import unittest + +from parameterized import parameterized + +from openpilot.selfdrive.test.process_replay.regen import regen_segment +from openpilot.selfdrive.test.process_replay.process_replay import check_openpilot_enabled, CONFIGS +from openpilot.selfdrive.test.openpilotci import get_url +from openpilot.tools.lib.logreader import LogReader +from openpilot.tools.lib.framereader import FrameReader + +EXCLUDED_PROCESSES = {"dmonitoringd", "dmonitoringmodeld"} +TESTED_SEGMENTS = [ + ("PRIUS_C2", "0982d79ebb0de295|2021-01-04--17-13-21--13"), # TOYOTA PRIUS 2017: NEO, pandaStateDEPRECATED, no peripheralState, sensorEventsDEPRECATED + # Enable these once regen on CI becomes faster or use them for different tests running controlsd in isolation + # ("MAZDA_C3", "bd6a637565e91581|2021-10-30--15-14-53--4"), # MAZDA.CX9_2021: TICI, incomplete managerState + # ("FORD_C3", "54827bf84c38b14f|2023-01-26--21-59-07--4"), # FORD.BRONCO_SPORT_MK1: TICI +] + + +def ci_setup_data_readers(route, sidx): + lr = LogReader(get_url(route, sidx, "rlog")) + # dm disabled + frs = { + 'roadCameraState': FrameReader(get_url(route, sidx, "fcamera")), + } + if next((True for m in lr if m.which() == "wideRoadCameraState"), False): + frs["wideRoadCameraState"] = FrameReader(get_url(route, sidx, "ecamera")) + + return lr, frs + + +class TestRegen(unittest.TestCase): + @parameterized.expand(TESTED_SEGMENTS) + def test_engaged(self, case_name, segment): + tested_procs = [p for p in CONFIGS if p.proc_name not in EXCLUDED_PROCESSES] + + route, sidx = segment.rsplit("--", 1) + lr, frs = ci_setup_data_readers(route, sidx) + output_logs = regen_segment(lr, frs, processes=tested_procs, disable_tqdm=True) + + engaged = check_openpilot_enabled(output_logs) + self.assertTrue(engaged, f"openpilot not engaged in {case_name}") + + +if __name__=='__main__': + unittest.main()