diff --git a/selfdrive/modeld/runners/onnx_runner.py b/selfdrive/modeld/runners/onnx_runner.py index 3b19550e8f..7447c33aa6 100755 --- a/selfdrive/modeld/runners/onnx_runner.py +++ b/selfdrive/modeld/runners/onnx_runner.py @@ -25,6 +25,11 @@ def write(d): def run_loop(m): ishapes = [[1]+ii.shape[1:] for ii in m.get_inputs()] keys = [x.name for x in m.get_inputs()] + + # run once to initialize CUDA runner if regenerating routes + if "CUDAExecutionProvider" in m.get_providers() and "ONNX_INIT_PREDICTION" in os.environ: + m.run(None, dict(zip(keys, [np.zeros(shp, dtype=np.float32) for shp in ishapes]))) + print("ready to run onnx model", keys, ishapes, file=sys.stderr) while 1: inputs = [] diff --git a/selfdrive/test/process_replay/regen.py b/selfdrive/test/process_replay/regen.py index d1beabaaaf..c4ec4be7d7 100755 --- a/selfdrive/test/process_replay/regen.py +++ b/selfdrive/test/process_replay/regen.py @@ -6,6 +6,7 @@ from tqdm import tqdm import argparse # run DM procs os.environ["USE_WEBCAM"] = "1" +os.environ["ONNX_INIT_PREDICTION"] = "1" import cereal.messaging as messaging from cereal.services import service_list @@ -218,6 +219,10 @@ def regen_segment(lr, frs=None, outdir=FAKEDATA): } try: + # modeld needs time to initialize due to onnx cuda runner + managed_processes["modeld"].start() + time.sleep(5) + # start procs up ignore = list(fake_daemons.keys()) + ['ui', 'manage_athenad', 'uploader'] ensure_running(managed_processes.values(), started=True, not_run=ignore) @@ -243,9 +248,9 @@ def regen_segment(lr, frs=None, outdir=FAKEDATA): del vs segment = params.get("CurrentRoute", encoding='utf-8') + "--0" - seg_path = os.path.join(outdir, segment, "rlog.bz2") + seg_path = os.path.join(outdir, segment) # check to make sure openpilot is engaged in the route - if not check_enabled(LogReader(seg_path)): + if not check_enabled(LogReader(os.path.join(seg_path, "rlog.bz2"))): raise Exception(f"Route never enabled: {segment}") return seg_path