process_replay: deprecated message migration (#28456)

* Move migration code to new file

* Migrate messages in process replay. Remove old code

* Fix old_logtime

* Keep old logtimes on migrated messages

* Update ref commit
old-commit-hash: 838346d8a3

selfdrive/test/process_replay/migration.py

@@ -0,0 +1,65 @@
+from cereal import messaging
+
+
+def migrate_all(lr, old_logtime=False):
+  msgs = migrate_sensorEvents(lr, old_logtime)
+  msgs = migrate_carParams(msgs, old_logtime)
+
+  return msgs
+
+
+def migrate_carParams(lr, old_logtime=False):
+  all_msgs = []
+  for msg in lr:
+    if msg.which() == 'carParams':
+      CP = messaging.new_message('carParams')
+      CP.carParams = msg.carParams.as_builder()
+      for car_fw in CP.carParams.carFw:
+        car_fw.brand = CP.carParams.carName
+      if old_logtime:
+        CP.logMonoTime = msg.logMonoTime
+      msg = CP.as_reader()
+    all_msgs.append(msg)
+
+  return all_msgs
+
+
+def migrate_sensorEvents(lr, old_logtime=False):
+  all_msgs = []
+  for msg in lr:
+    if msg.which() != 'sensorEventsDEPRECATED':
+      all_msgs.append(msg)
+      continue
+
+    # migrate to split sensor events
+    for evt in msg.sensorEventsDEPRECATED:
+      # build new message for each sensor type
+      sensor_service = ''
+      if evt.which() == 'acceleration':
+        sensor_service = 'accelerometer'
+      elif evt.which() == 'gyro' or evt.which() == 'gyroUncalibrated':
+        sensor_service = 'gyroscope'
+      elif evt.which() == 'light' or evt.which() == 'proximity':
+        sensor_service = 'lightSensor'
+      elif evt.which() == 'magnetic' or evt.which() == 'magneticUncalibrated':
+        sensor_service = 'magnetometer'
+      elif evt.which() == 'temperature':
+        sensor_service = 'temperatureSensor'
+
+      m = messaging.new_message(sensor_service)
+      m.valid = True
+      if old_logtime:
+        m.logMonoTime = msg.logMonoTime
+
+      m_dat = getattr(m, sensor_service)
+      m_dat.version = evt.version
+      m_dat.sensor = evt.sensor
+      m_dat.type = evt.type
+      m_dat.source = evt.source
+      if old_logtime:
+        m_dat.timestamp = evt.timestamp
+      setattr(m_dat, evt.which(), getattr(evt, evt.which()))
+
+      all_msgs.append(m.as_reader())
+
+  return all_msgs

selfdrive/test/process_replay/process_replay.py

@@ -18,6 +18,7 @@ from panda.python import ALTERNATIVE_EXPERIENCE
 from selfdrive.car.car_helpers import get_car, interfaces
 from selfdrive.manager.process_config import managed_processes
 from selfdrive.test.process_replay.helpers import OpenpilotPrefix
+from selfdrive.test.process_replay.migration import migrate_all
 
 # Numpy gives different results based on CPU features after version 19
 NUMPY_TOLERANCE = 1e-7
@@ -342,7 +343,7 @@ def replay_process_with_name(name, lr, *args, **kwargs):
 
 
 def replay_process(cfg, lr, fingerprint=None, return_all_logs=False, disable_progress=False):
-  all_msgs = list(lr)
+  all_msgs = migrate_all(lr, old_logtime=True)
   process_logs = _replay_single_process(cfg, all_msgs, fingerprint, disable_progress)
 
   if return_all_logs:

selfdrive/test/process_replay/ref_commit

@@ -1 +1 @@
-3e684aef4483b8d311d71bab3bb543d7bad26563
+67bcd498f0432252a3f11a9fd2b59b344ee214ba

selfdrive/test/process_replay/regen.py

@@ -20,6 +20,7 @@ from selfdrive.car.toyota.values import EPS_SCALE
 from selfdrive.manager.process import ensure_running
 from selfdrive.manager.process_config import managed_processes
 from selfdrive.test.process_replay.process_replay import CONFIGS, FAKEDATA, setup_env, check_openpilot_enabled
+from selfdrive.test.process_replay.migration import migrate_all
 from selfdrive.test.update_ci_routes import upload_route
 from tools.lib.route import Route
 from tools.lib.framereader import FrameReader
@@ -179,67 +180,11 @@ def replay_cameras(lr, frs, disable_tqdm=False):
   return vs, p
 
 
-def migrate_carparams(lr):
-  all_msgs = []
-  for msg in lr:
-    if msg.which() == 'carParams':
-      CP = messaging.new_message('carParams')
-      CP.carParams = msg.carParams.as_builder()
-      for car_fw in CP.carParams.carFw:
-        car_fw.brand = CP.carParams.carName
-      msg = CP.as_reader()
-    all_msgs.append(msg)
-
-  return all_msgs
-
-
-def migrate_sensorEvents(lr, old_logtime=False):
-  all_msgs = []
-  for msg in lr:
-    if msg.which() != 'sensorEventsDEPRECATED':
-      all_msgs.append(msg)
-      continue
-
-    # migrate to split sensor events
-    for evt in msg.sensorEventsDEPRECATED:
-      # build new message for each sensor type
-      sensor_service = ''
-      if evt.which() == 'acceleration':
-        sensor_service = 'accelerometer'
-      elif evt.which() == 'gyro' or evt.which() == 'gyroUncalibrated':
-        sensor_service = 'gyroscope'
-      elif evt.which() == 'light' or evt.which() == 'proximity':
-        sensor_service = 'lightSensor'
-      elif evt.which() == 'magnetic' or evt.which() == 'magneticUncalibrated':
-        sensor_service = 'magnetometer'
-      elif evt.which() == 'temperature':
-        sensor_service = 'temperatureSensor'
-
-      m = messaging.new_message(sensor_service)
-      m.valid = True
-      if old_logtime:
-        m.logMonoTime = msg.logMonoTime
-
-      m_dat = getattr(m, sensor_service)
-      m_dat.version = evt.version
-      m_dat.sensor = evt.sensor
-      m_dat.type = evt.type
-      m_dat.source = evt.source
-      if old_logtime:
-        m_dat.timestamp = evt.timestamp
-      setattr(m_dat, evt.which(), getattr(evt, evt.which()))
-
-      all_msgs.append(m.as_reader())
-
-  return all_msgs
-
-
 def regen_segment(lr, frs=None, daemons="all", outdir=FAKEDATA, disable_tqdm=False):
   if not isinstance(daemons, str) and not hasattr(daemons, "__iter__"):
     raise ValueError("whitelist_proc must be a string or iterable")
 
-  lr = migrate_carparams(list(lr))
+  lr = migrate_all(lr)
-  lr = migrate_sensorEvents(list(lr))
   if frs is None:
     frs = dict()