From 838346d8a3a139a160e3cc602ba38d6f3b09a00a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20R=C4=85czy?= Date: Sat, 10 Jun 2023 04:17:50 +0200 Subject: [PATCH] process_replay: deprecated message migration (#28456) * Move migration code to new file * Migrate messages in process replay. Remove old code * Fix old_logtime * Keep old logtimes on migrated messages * Update ref commit --- selfdrive/test/process_replay/migration.py | 65 +++++++++++++++++++ .../test/process_replay/process_replay.py | 3 +- selfdrive/test/process_replay/ref_commit | 2 +- selfdrive/test/process_replay/regen.py | 59 +---------------- 4 files changed, 70 insertions(+), 59 deletions(-) create mode 100644 selfdrive/test/process_replay/migration.py diff --git a/selfdrive/test/process_replay/migration.py b/selfdrive/test/process_replay/migration.py new file mode 100644 index 0000000000..31b3acde56 --- /dev/null +++ b/selfdrive/test/process_replay/migration.py @@ -0,0 +1,65 @@ +from cereal import messaging + + +def migrate_all(lr, old_logtime=False): + msgs = migrate_sensorEvents(lr, old_logtime) + msgs = migrate_carParams(msgs, old_logtime) + + return msgs + + +def migrate_carParams(lr, old_logtime=False): + all_msgs = [] + for msg in lr: + if msg.which() == 'carParams': + CP = messaging.new_message('carParams') + CP.carParams = msg.carParams.as_builder() + for car_fw in CP.carParams.carFw: + car_fw.brand = CP.carParams.carName + if old_logtime: + CP.logMonoTime = msg.logMonoTime + msg = CP.as_reader() + all_msgs.append(msg) + + return all_msgs + + +def migrate_sensorEvents(lr, old_logtime=False): + all_msgs = [] + for msg in lr: + if msg.which() != 'sensorEventsDEPRECATED': + all_msgs.append(msg) + continue + + # migrate to split sensor events + for evt in msg.sensorEventsDEPRECATED: + # build new message for each sensor type + sensor_service = '' + if evt.which() == 'acceleration': + sensor_service = 'accelerometer' + elif evt.which() == 'gyro' or evt.which() == 'gyroUncalibrated': + sensor_service = 'gyroscope' + elif evt.which() == 'light' or evt.which() == 'proximity': + sensor_service = 'lightSensor' + elif evt.which() == 'magnetic' or evt.which() == 'magneticUncalibrated': + sensor_service = 'magnetometer' + elif evt.which() == 'temperature': + sensor_service = 'temperatureSensor' + + m = messaging.new_message(sensor_service) + m.valid = True + if old_logtime: + m.logMonoTime = msg.logMonoTime + + m_dat = getattr(m, sensor_service) + m_dat.version = evt.version + m_dat.sensor = evt.sensor + m_dat.type = evt.type + m_dat.source = evt.source + if old_logtime: + m_dat.timestamp = evt.timestamp + setattr(m_dat, evt.which(), getattr(evt, evt.which())) + + all_msgs.append(m.as_reader()) + + return all_msgs diff --git a/selfdrive/test/process_replay/process_replay.py b/selfdrive/test/process_replay/process_replay.py index 9261aee838..22d1516373 100755 --- a/selfdrive/test/process_replay/process_replay.py +++ b/selfdrive/test/process_replay/process_replay.py @@ -18,6 +18,7 @@ from panda.python import ALTERNATIVE_EXPERIENCE from selfdrive.car.car_helpers import get_car, interfaces from selfdrive.manager.process_config import managed_processes from selfdrive.test.process_replay.helpers import OpenpilotPrefix +from selfdrive.test.process_replay.migration import migrate_all # Numpy gives different results based on CPU features after version 19 NUMPY_TOLERANCE = 1e-7 @@ -342,7 +343,7 @@ def replay_process_with_name(name, lr, *args, **kwargs): def replay_process(cfg, lr, fingerprint=None, return_all_logs=False, disable_progress=False): - all_msgs = list(lr) + all_msgs = migrate_all(lr, old_logtime=True) process_logs = _replay_single_process(cfg, all_msgs, fingerprint, disable_progress) if return_all_logs: diff --git a/selfdrive/test/process_replay/ref_commit b/selfdrive/test/process_replay/ref_commit index 2332054b69..f3a9408671 100644 --- a/selfdrive/test/process_replay/ref_commit +++ b/selfdrive/test/process_replay/ref_commit @@ -1 +1 @@ -3e684aef4483b8d311d71bab3bb543d7bad26563 +67bcd498f0432252a3f11a9fd2b59b344ee214ba diff --git a/selfdrive/test/process_replay/regen.py b/selfdrive/test/process_replay/regen.py index cc697bb9bf..18c7e3a8dc 100755 --- a/selfdrive/test/process_replay/regen.py +++ b/selfdrive/test/process_replay/regen.py @@ -20,6 +20,7 @@ from selfdrive.car.toyota.values import EPS_SCALE from selfdrive.manager.process import ensure_running from selfdrive.manager.process_config import managed_processes from selfdrive.test.process_replay.process_replay import CONFIGS, FAKEDATA, setup_env, check_openpilot_enabled +from selfdrive.test.process_replay.migration import migrate_all from selfdrive.test.update_ci_routes import upload_route from tools.lib.route import Route from tools.lib.framereader import FrameReader @@ -179,67 +180,11 @@ def replay_cameras(lr, frs, disable_tqdm=False): return vs, p -def migrate_carparams(lr): - all_msgs = [] - for msg in lr: - if msg.which() == 'carParams': - CP = messaging.new_message('carParams') - CP.carParams = msg.carParams.as_builder() - for car_fw in CP.carParams.carFw: - car_fw.brand = CP.carParams.carName - msg = CP.as_reader() - all_msgs.append(msg) - - return all_msgs - - -def migrate_sensorEvents(lr, old_logtime=False): - all_msgs = [] - for msg in lr: - if msg.which() != 'sensorEventsDEPRECATED': - all_msgs.append(msg) - continue - - # migrate to split sensor events - for evt in msg.sensorEventsDEPRECATED: - # build new message for each sensor type - sensor_service = '' - if evt.which() == 'acceleration': - sensor_service = 'accelerometer' - elif evt.which() == 'gyro' or evt.which() == 'gyroUncalibrated': - sensor_service = 'gyroscope' - elif evt.which() == 'light' or evt.which() == 'proximity': - sensor_service = 'lightSensor' - elif evt.which() == 'magnetic' or evt.which() == 'magneticUncalibrated': - sensor_service = 'magnetometer' - elif evt.which() == 'temperature': - sensor_service = 'temperatureSensor' - - m = messaging.new_message(sensor_service) - m.valid = True - if old_logtime: - m.logMonoTime = msg.logMonoTime - - m_dat = getattr(m, sensor_service) - m_dat.version = evt.version - m_dat.sensor = evt.sensor - m_dat.type = evt.type - m_dat.source = evt.source - if old_logtime: - m_dat.timestamp = evt.timestamp - setattr(m_dat, evt.which(), getattr(evt, evt.which())) - - all_msgs.append(m.as_reader()) - - return all_msgs - - def regen_segment(lr, frs=None, daemons="all", outdir=FAKEDATA, disable_tqdm=False): if not isinstance(daemons, str) and not hasattr(daemons, "__iter__"): raise ValueError("whitelist_proc must be a string or iterable") - lr = migrate_carparams(list(lr)) - lr = migrate_sensorEvents(list(lr)) + lr = migrate_all(lr) if frs is None: frs = dict()