regen: Refactor log migrate functions to avoid needing azure keys (#26049)

* refactor migrate fns to avoid needing to use azure keys on import

* move azure key init behind a function

* resolve comments
old-commit-hash: 5f7d9a519e

selfdrive/test/process_replay/regen.py

@@ -224,13 +224,14 @@ def migrate_sensorEvents(lr, old_logtime=False):
       m_dat.sensor = evt.sensor
       m_dat.type = evt.type
       m_dat.source = evt.source
+      if old_logtime:
+        m_dat.timestamp = evt.timestamp
       setattr(m_dat, evt.which(), getattr(evt, evt.which()))
 
       all_msgs.append(m.as_reader())
 
   return all_msgs
 
-
 def regen_segment(lr, frs=None, outdir=FAKEDATA, disable_tqdm=False):
   lr = migrate_carparams(list(lr))
   lr = migrate_sensorEvents(list(lr))

selfdrive/test/update_ci_routes.py

@@ -14,11 +14,16 @@ SOURCES = [
   (_DATA_ACCOUNT_CI, "commadataci"),
 ]
 
-DEST_KEY = azureutil.get_user_token(_DATA_ACCOUNT_CI, "openpilotci")
-SOURCE_KEYS = [azureutil.get_user_token(account, bucket) for account, bucket in SOURCES]
-SERVICE = BlockBlobService(_DATA_ACCOUNT_CI, sas_token=DEST_KEY)
+
+def get_azure_keys():
+  dest_key = azureutil.get_user_token(_DATA_ACCOUNT_CI, "openpilotci")
+  source_keys = [azureutil.get_user_token(account, bucket) for account, bucket in SOURCES]
+  service = BlockBlobService(_DATA_ACCOUNT_CI, sas_token=dest_key)
+  return dest_key, source_keys, service
+
 
 def upload_route(path, exclude_patterns=None):
+  dest_key, _, _ = get_azure_keys()
   if exclude_patterns is None:
     exclude_patterns = ['*/dcamera.hevc']
 
@@ -29,27 +34,28 @@ def upload_route(path, exclude_patterns=None):
     "azcopy",
     "copy",
     f"{path}/*",
-    f"https://{_DATA_ACCOUNT_CI}.blob.core.windows.net/openpilotci/{destpath}?{DEST_KEY}",
+    f"https://{_DATA_ACCOUNT_CI}.blob.core.windows.net/openpilotci/{destpath}?{dest_key}",
     "--recursive=false",
     "--overwrite=false",
   ] + [f"--exclude-pattern={p}" for p in exclude_patterns]
   subprocess.check_call(cmd)
 
 def sync_to_ci_public(route):
+  dest_key, source_keys, service = get_azure_keys()
   key_prefix = route.replace('|', '/')
   dongle_id = key_prefix.split('/')[0]
 
-  if next(azureutil.list_all_blobs(SERVICE, "openpilotci", prefix=key_prefix), None) is not None:
+  if next(azureutil.list_all_blobs(service, "openpilotci", prefix=key_prefix), None) is not None:
     return True
 
   print(f"Uploading {route}")
-  for (source_account, source_bucket), source_key in zip(SOURCES, SOURCE_KEYS):
+  for (source_account, source_bucket), source_key in zip(SOURCES, source_keys):
     print(f"Trying {source_account}/{source_bucket}")
     cmd = [
       "azcopy",
       "copy",
       f"https://{source_account}.blob.core.windows.net/{source_bucket}/{key_prefix}?{source_key}",
-      f"https://{_DATA_ACCOUNT_CI}.blob.core.windows.net/openpilotci/{dongle_id}?{DEST_KEY}",
+      f"https://{_DATA_ACCOUNT_CI}.blob.core.windows.net/openpilotci/{dongle_id}?{dest_key}",
       "--recursive=true",
       "--overwrite=false",
       "--exclude-pattern=*/dcamera.hevc",