From 942f4a3fcf2f36bc31c618457803c57f2de35959 Mon Sep 17 00:00:00 2001
From: "Quantizr (Jimmy)" <9859727+Quantizr@users.noreply.github.com>
Date: Fri, 22 Aug 2025 17:39:34 -0700
Subject: [PATCH 01/19] jotpluggler!

---
 pyproject.toml               |   1 +
 tools/jotpluggler/data.py    | 179 ++++++++++++++++++++
 tools/jotpluggler/layout.py  | 304 ++++++++++++++++++++++++++++++++++
 tools/jotpluggler/pluggle.py | 240 +++++++++++++++++++++++++++
 tools/jotpluggler/views.py   | 308 +++++++++++++++++++++++++++++++++++
 uv.lock                      |  83 ++++++----
 6 files changed, 1082 insertions(+), 33 deletions(-)
 create mode 100644 tools/jotpluggler/data.py
 create mode 100644 tools/jotpluggler/layout.py
 create mode 100644 tools/jotpluggler/pluggle.py
 create mode 100644 tools/jotpluggler/views.py

diff --git a/pyproject.toml b/pyproject.toml
index 7d6516c0fb..5225a727c3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -121,6 +121,7 @@ dev = [
 
 tools = [
   "metadrive-simulator @ https://github.com/commaai/metadrive/releases/download/MetaDrive-minimal-0.4.2.4/metadrive_simulator-0.4.2.4-py3-none-any.whl ; (platform_machine != 'aarch64')",
+  "dearpygui>=2.1.0",
 ]
 
 [project.urls]
diff --git a/tools/jotpluggler/data.py b/tools/jotpluggler/data.py
new file mode 100644
index 0000000000..2d15d23b7b
--- /dev/null
+++ b/tools/jotpluggler/data.py
@@ -0,0 +1,179 @@
+import threading
+import numpy as np
+from abc import ABC, abstractmethod
+from typing import Any
+from openpilot.tools.lib.logreader import LogReader
+from openpilot.tools.lib.log_time_series import msgs_to_time_series
+
+
+# TODO: support cereal/ZMQ streaming
+class DataSource(ABC):
+  @abstractmethod
+  def load_data(self) -> dict[str, Any]:
+    pass
+
+  @abstractmethod
+  def get_duration(self) -> float:
+    pass
+
+
+class LogReaderSource(DataSource):
+  def __init__(self, route_name: str):
+    self.route_name = route_name
+    self._duration = 0.0
+    self._start_time_mono = 0.0
+
+  def load_data(self) -> dict[str, Any]:
+    lr = LogReader(self.route_name)
+    raw_time_series = msgs_to_time_series(lr)
+    processed_data = self._expand_list_fields(raw_time_series)
+
+    # Calculate timing information
+    times = [data['t'] for data in processed_data.values() if 't' in data and len(data['t']) > 0]
+    if times:
+      all_times = np.concatenate(times)
+      self._start_time_mono = all_times.min()
+      self._duration = all_times.max() - self._start_time_mono
+
+    return {'time_series_data': processed_data, 'route_start_time_mono': self._start_time_mono, 'duration': self._duration}
+
+  def get_duration(self) -> float:
+    return self._duration
+
+  # TODO: lists are expanded, but lists of structs are not
+  def _expand_list_fields(self, time_series_data):
+    expanded_data = {}
+    for msg_type, data in time_series_data.items():
+      expanded_data[msg_type] = {}
+      for field, values in data.items():
+        if field == 't':
+          expanded_data[msg_type]['t'] = values
+          continue
+
+        if isinstance(values, np.ndarray) and values.dtype == object:  # ragged array
+          lens = np.fromiter((len(v) for v in values), dtype=int, count=len(values))
+          max_len = lens.max() if lens.size else 0
+          if max_len > 0:
+            arr = np.full((len(values), max_len), None, dtype=object)
+            for i, v in enumerate(values):
+              arr[i, : lens[i]] = v
+            for i in range(max_len):
+              sub_arr = arr[:, i]
+              expanded_data[msg_type][f"{field}/{i}"] = sub_arr
+        elif isinstance(values, np.ndarray) and values.ndim > 1:  # regular array
+          for i in range(values.shape[1]):
+            col_data = values[:, i]
+            expanded_data[msg_type][f"{field}/{i}"] = col_data
+        else:
+          expanded_data[msg_type][field] = values
+    return expanded_data
+
+
+class DataLoadedEvent:
+  def __init__(self, data: dict[str, Any]):
+    self.data = data
+
+
+class Observer(ABC):
+  @abstractmethod
+  def on_data_loaded(self, event: DataLoadedEvent):
+    pass
+
+
+class DataManager:
+  def __init__(self):
+    self.time_series_data = {}
+    self.loading = False
+    self.route_start_time_mono = 0.0
+    self.duration = 100.0
+    self._observers: list[Observer] = []
+
+  def add_observer(self, observer: Observer):
+    self._observers.append(observer)
+
+  def remove_observer(self, observer: Observer):
+    if observer in self._observers:
+      self._observers.remove(observer)
+
+  def _notify_observers(self, event: DataLoadedEvent):
+    for observer in self._observers:
+      observer.on_data_loaded(event)
+
+  def get_current_value_for_path(self, path: str, time_s: float, last_index: int | None = None):
+    try:
+      abs_time_s = self.route_start_time_mono + time_s
+      msg_type, field_path = path.split('/', 1)
+      ts_data = self.time_series_data[msg_type]
+      t, v = ts_data['t'], ts_data[field_path]
+
+      if len(t) == 0:
+        return None, None
+
+      if last_index is None:  # jump
+        idx = np.searchsorted(t, abs_time_s, side='right') - 1
+      else:  # continuous playback
+        idx = last_index
+        while idx < len(t) - 1 and t[idx + 1] < abs_time_s:
+          idx += 1
+
+      idx = max(0, idx)
+      return v[idx], idx
+
+    except (KeyError, IndexError):
+      return None, None
+
+  def get_all_paths(self) -> list[str]:
+    all_paths = []
+    for msg_type, data in self.time_series_data.items():
+      for key in data.keys():
+        if key != 't':
+          all_paths.append(f"{msg_type}/{key}")
+    return all_paths
+
+  def is_path_plottable(self, path: str) -> bool:
+    try:
+      msg_type, field_path = path.split('/', 1)
+      value_array = self.time_series_data.get(msg_type, {}).get(field_path)
+      if value_array is not None:  # only numbers and bools are plottable
+        return np.issubdtype(value_array.dtype, np.number) or np.issubdtype(value_array.dtype, np.bool_)
+    except (ValueError, KeyError):
+      pass
+    return False
+
+  def get_time_series_data(self, path: str) -> tuple | None:
+    try:
+      msg_type, field_path = path.split('/', 1)
+      ts_data = self.time_series_data[msg_type]
+      time_array = ts_data['t']
+      plot_values = ts_data[field_path]
+
+      if len(time_array) == 0:
+        return None
+
+      rel_time_array = time_array - self.route_start_time_mono
+      return rel_time_array, plot_values
+
+    except (KeyError, ValueError):
+      return None
+
+  def load_route(self, route_name: str):
+    if self.loading:
+      return
+
+    self.loading = True
+    data_source = LogReaderSource(route_name)
+    threading.Thread(target=self._load_in_background, args=(data_source,), daemon=True).start()
+
+  def _load_in_background(self, data_source: DataSource):
+    try:
+      data = data_source.load_data()
+      self.time_series_data = data['time_series_data']
+      self.route_start_time_mono = data['route_start_time_mono']
+      self.duration = data['duration']
+
+      self._notify_observers(DataLoadedEvent(data))
+
+    except Exception as e:
+      print(f"Error loading route: {e}")
+    finally:
+      self.loading = False
diff --git a/tools/jotpluggler/layout.py b/tools/jotpluggler/layout.py
new file mode 100644
index 0000000000..764477fbe8
--- /dev/null
+++ b/tools/jotpluggler/layout.py
@@ -0,0 +1,304 @@
+import uuid
+import dearpygui.dearpygui as dpg
+from abc import ABC, abstractmethod
+from openpilot.tools.jotpluggler.data import DataManager
+from openpilot.tools.jotpluggler.views import ViewPanel, TimeSeriesPanel
+
+
+class LayoutNode(ABC):
+  def __init__(self, node_id: str = None):
+    self.node_id = node_id or str(uuid.uuid4())
+    self.tag = None
+
+  @abstractmethod
+  def create_ui(self, parent_tag: str, width: int = -1, height: int = -1):
+    pass
+
+  @abstractmethod
+  def destroy_ui(self):
+    pass
+
+  @abstractmethod
+  def preserve_data(self):
+    pass
+
+
+class LeafNode(LayoutNode):
+  """Leaf node that contains a single ViewPanel with controls"""
+
+  def __init__(self, panel: ViewPanel, layout_manager=None, scale: float = 1.0, node_id: str = None):
+    super().__init__(node_id)
+    self.panel = panel
+    self.layout_manager = layout_manager
+    self.scale = scale
+
+  def preserve_data(self):
+    self.panel.preserve_data()
+
+  def create_ui(self, parent_tag: str, width: int = -1, height: int = -1):
+    """Create UI container with controls and panel"""
+    self.tag = f"leaf_{self.node_id}"
+
+    with dpg.child_window(tag=self.tag, parent=parent_tag, border=True, width=-1, height=-1, no_scrollbar=True):
+      # Control bar
+      with dpg.group(horizontal=True):
+        dpg.add_input_text(tag=f"title_{self.node_id}", default_value=self.panel.title, width=int(100 * self.scale), callback=self._on_title_change)
+        dpg.add_combo(
+          items=["Time Series"],  # "Camera", "Text Log", "Map View"],
+          tag=f"type_{self.node_id}",
+          default_value="Time Series",
+          width=int(100 * self.scale),
+          callback=self._on_type_change,
+        )
+        dpg.add_button(label="Clear", callback=self._clear, width=int(50 * self.scale))
+        dpg.add_button(label="Delete", callback=self._delete, width=int(50 * self.scale))
+        dpg.add_button(label="Split H", callback=lambda: self._split("horizontal"), width=int(50 * self.scale))
+        dpg.add_button(label="Split V", callback=lambda: self._split("vertical"), width=int(50 * self.scale))
+
+      dpg.add_separator()
+
+      # Panel content area
+      panel_area_tag = f"panel_area_{self.node_id}"
+      with dpg.child_window(tag=panel_area_tag, border=False, height=-1, width=-1, no_scrollbar=True):
+        self.panel.create_ui(panel_area_tag)
+
+  def destroy_ui(self):
+    if self.panel:
+      self.panel.destroy_ui()
+    if self.tag and dpg.does_item_exist(self.tag):
+      dpg.delete_item(self.tag)
+
+  def _on_title_change(self, sender, app_data):
+    self.panel.title = app_data
+
+  def _on_type_change(self, sender, app_data):
+    print(f"Panel type change requested: {app_data}")
+
+  def _split(self, orientation: str):
+    if self.layout_manager:
+      self.layout_manager.split_node(self, orientation)
+
+  def _clear(self):
+    if hasattr(self.panel, 'clear_all_series'):
+      self.panel.clear_all_series()
+
+  def _delete(self):
+    if self.layout_manager:
+      self.layout_manager.delete_node(self)
+
+
+class SplitterNode(LayoutNode):
+  """Splitter node that contains multiple child nodes"""
+
+  def __init__(self, children: list[LayoutNode], orientation: str = "horizontal", node_id: str = None):
+    super().__init__(node_id)
+    self.children = children if children else []
+    self.orientation = orientation
+    self.child_proportions = [1.0 / len(self.children) for _ in self.children] if self.children else []
+    self.child_container_tags = []  # Track container tags for resizing (different from child tag)
+
+  def preserve_data(self):
+    for child in self.children:
+      child.preserve_data()
+
+  def add_child(self, child: LayoutNode, index: int = None):
+    if index is None:
+      self.children.append(child)
+      self.child_proportions.append(0.0)
+    else:
+      self.children.insert(index, child)
+      self.child_proportions.insert(index, 0.0)
+    self._redistribute_proportions()
+
+  def remove_child(self, child: LayoutNode):
+    if child in self.children:
+      index = self.children.index(child)
+      self.children.remove(child)
+      self.child_proportions.pop(index)
+      child.destroy_ui()
+      if self.children:
+        self._redistribute_proportions()
+
+  def replace_child(self, old_child: LayoutNode, new_child: LayoutNode):
+    try:
+      index = self.children.index(old_child)
+      self.children[index] = new_child
+      return index
+    except ValueError:
+      return None
+
+  def _redistribute_proportions(self):
+    if self.children:
+      equal_proportion = 1.0 / len(self.children)
+      self.child_proportions = [equal_proportion for _ in self.children]
+
+  def resize_children(self):
+    if not self.tag or not dpg.does_item_exist(self.tag):
+      return
+
+    available_width, available_height = dpg.get_item_rect_size(dpg.get_item_parent(self.tag))
+
+    for i, container_tag in enumerate(self.child_container_tags):
+      if not dpg.does_item_exist(container_tag):
+        continue
+
+      proportion = self.child_proportions[i] if i < len(self.child_proportions) else (1.0 / len(self.children))
+
+      if self.orientation == "horizontal":
+        new_width = max(100, int(available_width * proportion))
+        dpg.configure_item(container_tag, width=new_width)
+      else:
+        new_height = max(100, int(available_height * proportion))
+        dpg.configure_item(container_tag, height=new_height)
+
+      child = self.children[i] if i < len(self.children) else None
+      if child and isinstance(child, SplitterNode):
+        child.resize_children()
+
+  def create_ui(self, parent_tag: str, width: int = -1, height: int = -1):
+    self.tag = f"splitter_{self.node_id}"
+    self.child_container_tags = []
+
+    if self.orientation == "horizontal":
+      with dpg.group(tag=self.tag, parent=parent_tag, horizontal=True):
+        for i, child in enumerate(self.children):
+          proportion = self.child_proportions[i]
+          child_width = max(100, int(width * proportion))
+          container_tag = f"child_container_{self.node_id}_{i}"
+          self.child_container_tags.append(container_tag)
+
+          with dpg.child_window(tag=container_tag, width=child_width, height=-1, border=False, no_scrollbar=True, resizable_x=False):
+            child.create_ui(container_tag, child_width, height)
+    else:
+      with dpg.group(tag=self.tag, parent=parent_tag):
+        for i, child in enumerate(self.children):
+          proportion = self.child_proportions[i]
+          child_height = max(100, int(height * proportion))
+          container_tag = f"child_container_{self.node_id}_{i}"
+          self.child_container_tags.append(container_tag)
+
+          with dpg.child_window(tag=container_tag, width=-1, height=child_height, border=False, no_scrollbar=True, resizable_y=False):
+            child.create_ui(container_tag, width, child_height)
+
+  def destroy_ui(self):
+    for child in self.children:
+      if child:
+        child.destroy_ui()
+    if self.tag and dpg.does_item_exist(self.tag):
+      dpg.delete_item(self.tag)
+    self.child_container_tags.clear()
+
+
+class PlotLayoutManager:
+  def __init__(self, data_manager: DataManager, playback_manager, scale: float = 1.0):
+    self.data_manager = data_manager
+    self.playback_manager = playback_manager
+    self.scale = scale
+    self.root_node: LayoutNode = None
+    self.container_tag = "plot_layout_container"
+    self._initialize_default_layout()
+
+  def _initialize_default_layout(self):
+    panel = TimeSeriesPanel(self.data_manager, self.playback_manager)
+    self.root_node = LeafNode(panel, layout_manager=self, scale=self.scale)
+
+  def create_ui(self, parent_tag: str):
+    if dpg.does_item_exist(self.container_tag):
+      dpg.delete_item(self.container_tag)
+
+    with dpg.child_window(tag=self.container_tag, parent=parent_tag, border=False, width=-1, height=-1, no_scrollbar=True):
+      if self.root_node:
+        container_width, container_height = dpg.get_item_rect_size(self.container_tag)
+        self.root_node.create_ui(self.container_tag, container_width, container_height)
+
+  def on_viewport_resize(self):
+    if isinstance(self.root_node, SplitterNode):
+      self.root_node.resize_children()
+
+  def split_node(self, node: LeafNode, orientation: str):
+    # create new panel for the split
+    new_panel = TimeSeriesPanel(self.data_manager, self.playback_manager)  # TODO: create same type of panel as the split
+    new_leaf = LeafNode(new_panel, layout_manager=self, scale=self.scale)
+
+    parent_node, child_index = self._find_parent_and_index(node)
+
+    if parent_node is None:  # root node - create new splitter as root
+      node.preserve_data()
+      self.root_node = SplitterNode([node, new_leaf], orientation)
+      self._update_ui_for_node(self.root_node, self.container_tag)
+    elif isinstance(parent_node, SplitterNode) and parent_node.orientation == orientation:  # same orientation - add to existing splitter
+      parent_node.add_child(new_leaf, child_index + 1)
+      self._update_ui_for_node(parent_node)
+    else:  # different orientation - replace node with new splitter
+      node.preserve_data()
+      new_splitter = SplitterNode([node, new_leaf], orientation)
+      self._replace_child_in_parent(parent_node, node, new_splitter)
+
+  def delete_node(self, node: LeafNode):  # TODO: actually delete the node, not just the ui for the node
+    parent_node, child_index = self._find_parent_and_index(node)
+
+    if parent_node is None:  # root deletion - replace with new default
+      node.destroy_ui()
+      self._initialize_default_layout()
+      self._update_ui_for_node(self.root_node, self.container_tag)
+    elif isinstance(parent_node, SplitterNode):
+      parent_node.remove_child(node)
+      if len(parent_node.children) == 1:  # collapse splitter --> leaf to just leaf
+        remaining_child = parent_node.children[0]
+        grandparent_node, parent_index = self._find_parent_and_index(parent_node)
+
+        if grandparent_node is None:  # promote remaining child to root
+          remaining_child.preserve_data()
+          parent_node.children.remove(remaining_child)
+          self.root_node = remaining_child
+          parent_node.destroy_ui()
+          self._update_ui_for_node(self.root_node, self.container_tag)
+        else:  # replace splitter with remaining child in grandparent node
+          self._replace_child_in_parent(grandparent_node, parent_node, remaining_child)
+      else:  # update splpitter contents
+        self._update_ui_for_node(parent_node)
+
+  def _replace_child_in_parent(self, parent_node: SplitterNode, old_child: LayoutNode, new_child: LayoutNode):
+    old_child.preserve_data()  # save data and for when recreating ui for the node
+
+    child_index = parent_node.children.index(old_child)
+    child_container_tag = f"child_container_{parent_node.node_id}_{child_index}"
+
+    parent_node.replace_child(old_child, new_child)
+
+    # Clean up old child if it's being replaced (not just moved)
+    if old_child != new_child:
+      old_child.destroy_ui()
+
+    if dpg.does_item_exist(child_container_tag):
+      dpg.delete_item(child_container_tag, children_only=True)
+      container_width, container_height = dpg.get_item_rect_size(child_container_tag)
+      new_child.create_ui(child_container_tag, container_width, container_height)
+
+  def _update_ui_for_node(self, node: LayoutNode, container_tag: str = None):
+    node.preserve_data()
+
+    if container_tag:  #  update node in a specific container (usually root)
+      dpg.delete_item(container_tag, children_only=True)
+      container_width, container_height = dpg.get_item_rect_size(container_tag)
+      node.create_ui(container_tag, container_width, container_height)
+    else:  # update node in its current location (splitter updates)
+      if node.tag and dpg.does_item_exist(node.tag):
+        parent_container = dpg.get_item_parent(node.tag)
+        node.destroy_ui()
+        if parent_container and dpg.does_item_exist(parent_container):
+          parent_width, parent_height = dpg.get_item_rect_size(parent_container)
+          node.create_ui(parent_container, parent_width, parent_height)
+
+  def _find_parent_and_index(self, target_node: LayoutNode) -> tuple:  # TODO: probably can be stored in child
+    def search_recursive(node: LayoutNode, parent: LayoutNode = None, index: int = 0):
+      if node == target_node:
+        return parent, index
+      if isinstance(node, SplitterNode):
+        for i, child in enumerate(node.children):
+          result = search_recursive(child, node, i)
+          if result[0] is not None:
+            return result
+      return None, None
+
+    return search_recursive(self.root_node)
diff --git a/tools/jotpluggler/pluggle.py b/tools/jotpluggler/pluggle.py
new file mode 100644
index 0000000000..955009c083
--- /dev/null
+++ b/tools/jotpluggler/pluggle.py
@@ -0,0 +1,240 @@
+import argparse
+import pyautogui
+import subprocess
+import dearpygui.dearpygui as dpg
+import threading
+from openpilot.tools.jotpluggler.data import DataManager, Observer, DataLoadedEvent
+from openpilot.tools.jotpluggler.views import DataTreeView
+from openpilot.tools.jotpluggler.layout import PlotLayoutManager, SplitterNode, LeafNode
+
+
+class PlaybackManager:
+  def __init__(self):
+    self.is_playing = False
+    self.current_time_s = 0.0
+    self.duration_s = 100.0
+    self.last_indices = {}
+
+  def set_route_duration(self, duration: float):
+    self.duration_s = duration
+    self.seek(min(self.current_time_s, duration))
+
+  def toggle_play_pause(self):
+    if not self.is_playing and self.current_time_s >= self.duration_s:
+      self.seek(0.0)
+    self.is_playing = not self.is_playing
+
+  def seek(self, time_s: float):
+    self.is_playing = False
+    self.current_time_s = max(0.0, min(time_s, self.duration_s))
+    self.last_indices.clear()
+
+  def update_time(self, delta_t: float) -> float:
+    if self.is_playing:
+      self.current_time_s = min(self.current_time_s + delta_t, self.duration_s)
+      if self.current_time_s >= self.duration_s:
+        self.is_playing = False
+    return self.current_time_s
+
+  def update_index(self, path: str, new_idx: int | None):
+    if new_idx is not None:
+      self.last_indices[path] = new_idx
+
+
+def calculate_avg_char_width(font):
+  sample_text = "abcdefghijklmnopqrstuvwxyz0123456789"
+  if size := dpg.get_text_size(sample_text, font=font):
+    return size[0] / len(sample_text)
+  return None
+
+
+def format_and_truncate(value, available_width: float, avg_char_width: float) -> str:
+  s = str(value)
+  max_chars = int(available_width / avg_char_width) - 3
+  if len(s) > max_chars:
+    return s[: max(0, max_chars)] + "..."
+  return s
+
+
+class MainController(Observer):
+  def __init__(self, scale: float = 1.0):
+    self.ui_lock = threading.Lock()
+    self.scale = scale
+    self.data_manager = DataManager()
+    self.playback_manager = PlaybackManager()
+    self._create_global_themes()
+    self.data_tree_view = DataTreeView(self.data_manager, self.ui_lock)
+    self.plot_layout_manager = PlotLayoutManager(self.data_manager, self.playback_manager, scale=self.scale)
+    self.data_manager.add_observer(self)
+    self.avg_char_width = None
+
+  def _create_global_themes(self):
+    with dpg.theme(tag="global_line_theme"):
+      with dpg.theme_component(dpg.mvLineSeries):
+        scaled_thickness = max(1.0, self.scale)
+        dpg.add_theme_style(dpg.mvPlotStyleVar_LineWeight, scaled_thickness, category=dpg.mvThemeCat_Plots)
+
+    with dpg.theme(tag="global_timeline_theme"):
+      with dpg.theme_component(dpg.mvInfLineSeries):
+        scaled_thickness = max(1.0, self.scale)
+        dpg.add_theme_style(dpg.mvPlotStyleVar_LineWeight, scaled_thickness, category=dpg.mvThemeCat_Plots)
+        dpg.add_theme_color(dpg.mvPlotCol_Line, (255, 0, 0, 128), category=dpg.mvThemeCat_Plots)
+
+  def on_data_loaded(self, event: DataLoadedEvent):
+    self.playback_manager.set_route_duration(event.data['duration'])
+    num_msg_types = len(event.data['time_series_data'])
+    dpg.set_value("load_status", f"Loaded {num_msg_types} message types")
+    dpg.configure_item("load_button", enabled=True)
+    dpg.configure_item("timeline_slider", max_value=event.data['duration'])
+
+  def setup_ui(self):
+    with dpg.item_handler_registry(tag="tree_node_handler"):
+      dpg.add_item_toggled_open_handler(callback=self.data_tree_view.update_active_nodes_list)
+
+    dpg.set_viewport_resize_callback(callback=self.on_viewport_resize)
+
+    with dpg.window(tag="Primary Window"):
+      with dpg.group(horizontal=True):
+        # Left panel - Data tree
+        with dpg.child_window(label="Data Pool", width=300 * self.scale, tag="data_pool_window", border=True, resizable_x=True):
+          with dpg.group(horizontal=True):
+            dpg.add_input_text(tag="route_input", width=-75 * self.scale, hint="Enter route name...")
+            dpg.add_button(label="Load", callback=self.load_route, tag="load_button", width=-1)
+          dpg.add_text("Ready to load route", tag="load_status")
+          dpg.add_separator()
+          dpg.add_text("Available Data")
+          dpg.add_separator()
+          dpg.add_input_text(tag="search_input", width=-1, hint="Search fields...", callback=self.search_data)
+          dpg.add_separator()
+          with dpg.group(tag="data_tree_container", track_offset=True):
+            pass
+
+        # Right panel - Plots and timeline
+        with dpg.group():
+          with dpg.child_window(label="Plot Window", border=True, height=-(30 + 13 * self.scale), tag="main_plot_area"):
+            self.plot_layout_manager.create_ui("main_plot_area")
+
+          with dpg.child_window(label="Timeline", border=True):
+            with dpg.table(header_row=False, borders_innerH=False, borders_innerV=False, borders_outerH=False, borders_outerV=False):
+              dpg.add_table_column(width_fixed=True, init_width_or_weight=int(50 * self.scale))  # Play button
+              dpg.add_table_column(width_stretch=True)  # Timeline slider
+              dpg.add_table_column(width_fixed=True, init_width_or_weight=int(50 * self.scale))  # FPS counter
+              with dpg.table_row():
+                dpg.add_button(label="Play", tag="play_pause_button", callback=self.toggle_play_pause, width=int(50 * self.scale))
+                dpg.add_slider_float(tag="timeline_slider", default_value=0.0, label="", width=-1, callback=self.timeline_drag)
+                dpg.add_text("", tag="fps_counter")
+
+    dpg.set_primary_window("Primary Window", True)
+
+  def on_viewport_resize(self):
+    self.plot_layout_manager.on_viewport_resize()
+
+  def load_route(self):
+    route_name = dpg.get_value("route_input").strip()
+    if route_name:
+      dpg.set_value("load_status", "Loading route...")
+      dpg.configure_item("load_button", enabled=False)
+      self.data_manager.load_route(route_name)
+
+  def search_data(self):
+    search_term = dpg.get_value("search_input")
+    self.data_tree_view.search_data(search_term)
+
+  def toggle_play_pause(self, sender):
+    self.playback_manager.toggle_play_pause()
+    label = "Pause" if self.playback_manager.is_playing else "Play"
+    dpg.configure_item(sender, label=label)
+
+  def timeline_drag(self, sender, app_data):
+    self.playback_manager.seek(app_data)
+    dpg.configure_item("play_pause_button", label="Play")
+
+  def update_frame(self, font):
+    with self.ui_lock:
+      if self.avg_char_width is None:
+        self.avg_char_width = calculate_avg_char_width(font)  # must be calculated after first frame
+
+      new_time = self.playback_manager.update_time(dpg.get_delta_time())
+      if not dpg.is_item_active("timeline_slider"):
+        dpg.set_value("timeline_slider", new_time)
+
+      self._update_timeline_indicators(new_time)
+      if not self.data_manager.loading and self.avg_char_width:
+        self._update_data_values()
+
+      dpg.set_value("fps_counter", f"{dpg.get_frame_rate():.1f} FPS")
+
+  def _update_data_values(self):
+    pool_width = dpg.get_item_rect_size("data_pool_window")[0]
+    value_column_width = pool_width * 0.5
+    active_nodes = self.data_tree_view.active_leaf_nodes
+
+    for node in active_nodes:
+      path = node.full_path
+      value_tag = f"value_{path}"
+
+      if dpg.does_item_exist(value_tag) and dpg.is_item_visible(value_tag):
+        last_index = self.playback_manager.last_indices.get(path)
+        value, new_idx = self.data_manager.get_current_value_for_path(path, self.playback_manager.current_time_s, last_index)
+
+        if value is not None:
+          self.playback_manager.update_index(path, new_idx)
+          formatted_value = format_and_truncate(value, value_column_width, self.avg_char_width)
+          dpg.set_value(value_tag, formatted_value)
+
+  def _update_timeline_indicators(self, current_time_s: float):
+    def update_node_recursive(node):
+      if isinstance(node, LeafNode):
+        if hasattr(node.panel, 'update_timeline_indicator'):
+          node.panel.update_timeline_indicator(current_time_s)
+      elif isinstance(node, SplitterNode):
+        for child in node.children:
+          update_node_recursive(child)
+
+    if self.plot_layout_manager.root_node:
+      update_node_recursive(self.plot_layout_manager.root_node)
+
+
+def main(route_to_load=None):
+  dpg.create_context()
+
+  # TODO: find better way of calculating display scaling
+  try:
+    w, h = next(tuple(map(int, l.split()[0].split('x'))) for l in subprocess.check_output(['xrandr']).decode().split('\n') if '*' in l)  # actual resolution
+    scale = pyautogui.size()[0] / w  # scaled resolution
+  except Exception:
+    scale = 1
+
+  with dpg.font_registry():
+    default_font = dpg.add_font("selfdrive/assets/fonts/Inter-Regular.ttf", int(13 * scale))
+  dpg.bind_font(default_font)
+
+  viewport_width, viewport_height = int(1200 * scale), int(800 * scale)
+  mouse_x, mouse_y = pyautogui.position()  # TODO: find better way of creating the window where the user is (default dpg behavior annoying on multiple displays)
+  dpg.create_viewport(
+    title='JotPluggler', width=viewport_width, height=viewport_height, x_pos=mouse_x - viewport_width // 2, y_pos=mouse_y - viewport_height // 2
+  )
+  dpg.setup_dearpygui()
+
+  controller = MainController(scale=scale)
+  controller.setup_ui()
+
+  if route_to_load:
+    dpg.set_value("route_input", route_to_load)
+    controller.load_route()
+
+  dpg.show_viewport()
+
+  # Main loop
+  while dpg.is_dearpygui_running():
+    controller.update_frame(default_font)
+    dpg.render_dearpygui_frame()
+
+  dpg.destroy_context()
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="A tool for visualizing openpilot logs.")
+  parser.add_argument("route", nargs='?', default=None, help="Optional route name to load on startup.")
+  args = parser.parse_args()
+  main(route_to_load=args.route)
diff --git a/tools/jotpluggler/views.py b/tools/jotpluggler/views.py
new file mode 100644
index 0000000000..163f60e647
--- /dev/null
+++ b/tools/jotpluggler/views.py
@@ -0,0 +1,308 @@
+import os
+import re
+import uuid
+import threading
+import dearpygui.dearpygui as dpg
+from abc import ABC, abstractmethod
+from openpilot.tools.jotpluggler.data import Observer, DataLoadedEvent, DataManager
+
+
+class ViewPanel(ABC):
+  """Abstract base class for all view panels that can be displayed in a plot container"""
+
+  def __init__(self, panel_id: str = None):
+    self.panel_id = panel_id or str(uuid.uuid4())
+    self.title = "Untitled Panel"
+
+  @abstractmethod
+  def create_ui(self, parent_tag: str):
+    pass
+
+  @abstractmethod
+  def destroy_ui(self):
+    pass
+
+  @abstractmethod
+  def get_panel_type(self) -> str:
+    pass
+
+  @abstractmethod
+  def preserve_data(self):
+    pass
+
+
+class TimeSeriesPanel(ViewPanel, Observer):
+  def __init__(self, data_manager: DataManager, playback_manager, panel_id: str = None):
+    super().__init__(panel_id)
+    self.data_manager = data_manager
+    self.playback_manager = playback_manager
+    self.title = "Time Series Plot"
+    self.plotted_series: set[str] = set()
+    self.plot_tag = None
+    self.x_axis_tag = None
+    self.y_axis_tag = None
+    self.timeline_indicator_tag = None
+    self._ui_created = False
+
+    # Store series data for restoration and legend management
+    self._preserved_series_data = []  # TODO: the way we do this right now doesn't make much sense
+    self._series_legend_tags = {}  # Maps series_path to legend tag
+
+    self.data_manager.add_observer(self)
+
+  def preserve_data(self):
+    self._preserved_series_data = []
+    if self.plotted_series and self._ui_created:
+      for series_path in self.plotted_series:
+        time_value_data = self.data_manager.get_time_series_data(series_path)
+        if time_value_data:
+          self._preserved_series_data.append((series_path, time_value_data))
+
+  def create_ui(self, parent_tag: str):
+    self.plot_tag = f"plot_{self.panel_id}"
+    self.x_axis_tag = f"{self.plot_tag}_x_axis"
+    self.y_axis_tag = f"{self.plot_tag}_y_axis"
+    self.timeline_indicator_tag = f"{self.plot_tag}_timeline"
+
+    with dpg.plot(height=-1, width=-1, tag=self.plot_tag, parent=parent_tag, drop_callback=self._on_series_drop, payload_type="TIMESERIES_PAYLOAD"):
+      dpg.add_plot_legend()
+      dpg.add_plot_axis(dpg.mvXAxis, label="", tag=self.x_axis_tag)
+      dpg.add_plot_axis(dpg.mvYAxis, label="", tag=self.y_axis_tag)
+
+      timeline_series_tag = dpg.add_inf_line_series(x=[0], label="Timeline", parent=self.y_axis_tag, tag=self.timeline_indicator_tag)
+      dpg.bind_item_theme(timeline_series_tag, "global_timeline_theme")
+
+    # Restore series from preserved data
+    if self._preserved_series_data:
+      self.plotted_series.clear()
+      for series_path, (rel_time_array, value_array) in self._preserved_series_data:
+        self._add_series_with_data(series_path, rel_time_array, value_array)
+      self._preserved_series_data = []
+
+    self._ui_created = True
+
+  def update_timeline_indicator(self, current_time_s: float):
+    if not self._ui_created or not dpg.does_item_exist(self.timeline_indicator_tag):
+      return
+
+    dpg.set_value(self.timeline_indicator_tag, [[current_time_s], [0]])  # vertical line position
+
+    if self.plotted_series:  # update legend labels with current values
+      for series_path in self.plotted_series:
+        last_index = self.playback_manager.last_indices.get(series_path)
+        value, new_idx = self.data_manager.get_current_value_for_path(series_path, current_time_s, last_index)
+
+        if value is not None:
+          self.playback_manager.update_index(series_path, new_idx)
+
+          if isinstance(value, (int, float)):
+            if isinstance(value, float):
+              formatted_value = f"{value:.4f}" if abs(value) < 1000 else f"{value:.3e}"
+            else:
+              formatted_value = str(value)
+          else:
+            formatted_value = str(value)
+
+          # Update the series label to include current value
+          series_tag = f"series_{self.panel_id}_{series_path.replace('/', '_')}"
+          legend_label = f"{series_path}: {formatted_value}"
+
+          if dpg.does_item_exist(series_tag):
+            dpg.configure_item(series_tag, label=legend_label)
+
+  def _add_series_with_data(self, series_path: str, rel_time_array, value_array):
+    if series_path in self.plotted_series:
+      return False
+
+    series_tag = f"series_{self.panel_id}_{series_path.replace('/', '_')}"
+    line_series_tag = dpg.add_line_series(x=rel_time_array.tolist(), y=value_array.tolist(), label=series_path, parent=self.y_axis_tag, tag=series_tag)
+
+    dpg.bind_item_theme(line_series_tag, "global_line_theme")
+
+    self.plotted_series.add(series_path)
+    dpg.fit_axis_data(self.x_axis_tag)
+    dpg.fit_axis_data(self.y_axis_tag)
+    return True
+
+  def destroy_ui(self):
+    if self.plot_tag and dpg.does_item_exist(self.plot_tag):
+      dpg.delete_item(self.plot_tag)
+
+    # self.data_manager.remove_observer(self)
+    self._series_legend_tags.clear()
+    self._ui_created = False
+
+  def get_panel_type(self) -> str:
+    return "timeseries"
+
+  def add_series(self, series_path: str) -> bool:
+    if series_path in self.plotted_series:
+      return False
+
+    time_value_data = self.data_manager.get_time_series_data(series_path)
+    if time_value_data is None:
+      return False
+
+    rel_time_array, value_array = time_value_data
+    return self._add_series_with_data(series_path, rel_time_array, value_array)
+
+  def clear_all_series(self):
+    for series_path in self.plotted_series.copy():
+      self.remove_series(series_path)
+
+  def remove_series(self, series_path: str):
+    if series_path in self.plotted_series:
+      series_tag = f"series_{self.panel_id}_{series_path.replace('/', '_')}"
+      if dpg.does_item_exist(series_tag):
+        dpg.delete_item(series_tag)
+      self.plotted_series.remove(series_path)
+      if series_path in self._series_legend_tags:  # Clean up legend tag mapping
+        del self._series_legend_tags[series_path]
+
+  def on_data_loaded(self, event: DataLoadedEvent):
+    for series_path in self.plotted_series.copy():
+      self._update_series_data(series_path)
+
+  def _update_series_data(self, series_path: str):
+    time_value_data = self.data_manager.get_time_series_data(series_path)
+    if time_value_data is None:
+      return False
+
+    rel_time_array, value_array = time_value_data
+    series_tag = f"series_{self.panel_id}_{series_path.replace('/', '_')}"
+
+    if dpg.does_item_exist(series_tag):
+      dpg.set_value(series_tag, [rel_time_array.tolist(), value_array.tolist()])
+      dpg.fit_axis_data(self.x_axis_tag)
+      dpg.fit_axis_data(self.y_axis_tag)
+      return True
+    else:
+      self.plotted_series.discard(series_path)
+      return False
+
+  def _on_series_drop(self, sender, app_data, user_data):
+    series_path = app_data
+    self.add_series(series_path)
+
+
+class DataTreeNode:
+  def __init__(self, name: str, full_path: str = ""):
+    self.name = name
+    self.full_path = full_path
+    self.children = {}
+    self.is_leaf = False
+
+
+class DataTreeView(Observer):
+  def __init__(self, data_manager: DataManager, ui_lock: threading.Lock):
+    self.data_manager = data_manager
+    self.ui_lock = ui_lock
+    self.current_search = ""
+    self.data_tree = DataTreeNode(name="root")
+    self.active_leaf_nodes = []
+    self.data_manager.add_observer(self)
+
+  def on_data_loaded(self, event: DataLoadedEvent):
+    with self.ui_lock:
+      self.populate_data_tree()
+
+  def populate_data_tree(self):
+    if not dpg.does_item_exist("data_tree_container"):
+      return
+
+    dpg.delete_item("data_tree_container", children_only=True)
+    search_term = self.current_search.strip().lower()
+
+    self.data_tree = DataTreeNode(name="root")
+    all_paths = self.data_manager.get_all_paths()
+
+    for path in sorted(all_paths):
+      if not self._should_display_path(path, search_term):
+        continue
+
+      parts = path.split('/')
+      current_node = self.data_tree
+      current_path_prefix = ""
+
+      for part in parts:
+        current_path_prefix = f"{current_path_prefix}/{part}" if current_path_prefix else part
+        if part not in current_node.children:
+          current_node.children[part] = DataTreeNode(name=part, full_path=current_path_prefix)
+        current_node = current_node.children[part]
+
+      current_node.is_leaf = True
+
+    self._create_ui_from_tree_recursive(self.data_tree, "data_tree_container", search_term)
+    self.update_active_nodes_list()
+
+  def _should_display_path(self, path: str, search_term: str) -> bool:
+    if 'DEPRECATED' in path and not os.environ.get('SHOW_DEPRECATED'):
+      return False
+    return not search_term or search_term in path.lower()
+
+  def _natural_sort_key(self, node: DataTreeNode):
+    node_type_key = node.is_leaf
+    parts = [int(p) if p.isdigit() else p.lower() for p in re.split(r'(\d+)', node.name) if p]
+    return (node_type_key, parts)
+
+  def _create_ui_from_tree_recursive(self, node: DataTreeNode, parent_tag: str, search_term: str):
+    sorted_children = sorted(node.children.values(), key=self._natural_sort_key)
+
+    for child in sorted_children:
+      if child.is_leaf:
+        is_plottable = self.data_manager.is_path_plottable(child.full_path)
+
+        # Create draggable item
+        with dpg.group(parent=parent_tag) as draggable_group:
+          with dpg.table(header_row=False, borders_innerV=False, borders_outerH=False, borders_outerV=False, policy=dpg.mvTable_SizingStretchProp):
+            dpg.add_table_column(init_width_or_weight=0.5)
+            dpg.add_table_column(init_width_or_weight=0.5)
+            with dpg.table_row():
+              dpg.add_text(child.name)
+              dpg.add_text("N/A", tag=f"value_{child.full_path}")
+
+        # Add drag payload if plottable
+        if is_plottable:
+          with dpg.drag_payload(parent=draggable_group, drag_data=child.full_path, payload_type="TIMESERIES_PAYLOAD"):
+            dpg.add_text(f"Plot: {child.full_path}")
+
+      else:
+        node_tag = f"tree_{child.full_path}"
+        label = child.name
+
+        if '/' not in child.full_path:
+          sample_count = len(self.data_manager.time_series_data.get(child.full_path, {}).get('t', []))
+          label = f"{child.name} ({sample_count} samples)"
+
+        should_open = bool(search_term) and len(search_term) > 1 and any(search_term in path for path in self._get_all_descendant_paths(child))
+
+        with dpg.tree_node(label=label, parent=parent_tag, tag=node_tag, default_open=should_open):
+          dpg.bind_item_handler_registry(node_tag, "tree_node_handler")
+          self._create_ui_from_tree_recursive(child, node_tag, search_term)
+
+  def _get_all_descendant_paths(self, node: DataTreeNode):
+    for child_name, child_node in node.children.items():
+      child_name_lower = child_name.lower()
+      if child_node.is_leaf:
+        yield child_name_lower
+      else:
+        for path in self._get_all_descendant_paths(child_node):
+          yield f"{child_name_lower}/{path}"
+
+  def search_data(self, search_term: str):
+    self.current_search = search_term
+    self.populate_data_tree()
+
+  def update_active_nodes_list(self, sender=None, app_data=None, user_data=None):
+    self.active_leaf_nodes = self.get_active_leaf_nodes(self.data_tree)
+
+  def get_active_leaf_nodes(self, node: DataTreeNode):
+    active_leaves = []
+    for child in node.children.values():
+      if child.is_leaf:
+        active_leaves.append(child)
+      else:
+        node_tag = f"tree_{child.full_path}"
+        if dpg.does_item_exist(node_tag) and dpg.get_value(node_tag):
+          active_leaves.extend(self.get_active_leaf_nodes(child))
+    return active_leaves
diff --git a/uv.lock b/uv.lock
index e1678029cf..3d83e86bb6 100644
--- a/uv.lock
+++ b/uv.lock
@@ -451,6 +451,21 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/d2/fc/c0a3f4c4eaa5a22fbef91713474666e13d0ea2a69c84532579490a9f2cc8/dbus_next-0.2.3-py3-none-any.whl", hash = "sha256:58948f9aff9db08316734c0be2a120f6dc502124d9642f55e90ac82ffb16a18b", size = 57885, upload-time = "2021-07-25T22:11:25.466Z" },
 ]
 
+[[package]]
+name = "dearpygui"
+version = "2.1.0"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/92/fe/66293fc40254a29f060efd3398f2b1001ed79263ae1837db9ec42caa8f1d/dearpygui-2.1.0-cp311-cp311-macosx_10_6_x86_64.whl", hash = "sha256:03e5dc0b3dd2f7965e50bbe41f3316a814408064b582586de994d93afedb125c", size = 2100924, upload-time = "2025-07-07T14:20:00.602Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/4d/9fa1c3156ba7bbf4dc89e2e322998752fccfdc3575923a98dd6a4da48911/dearpygui-2.1.0-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:b5b37710c3fa135c48e2347f39ecd1f415146e86db5d404707a0bf72d16bd304", size = 1874441, upload-time = "2025-07-07T14:20:09.165Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/3c/af5673b50699e1734296a0b5bcef39bb6989175b001ad1f9b0e7888ad90d/dearpygui-2.1.0-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:b0cfd7ac7eaa090fc22d6aa60fc4b527fc631cee10c348e4d8df92bb39af03d2", size = 2636574, upload-time = "2025-07-07T14:20:14.951Z" },
+    { url = "https://files.pythonhosted.org/packages/7f/db/ed4db0bb3d88e7a8c405472641419086bef9632c4b8b0489dc0c43519c0d/dearpygui-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:a9af54f96d3ef30c5db9d12cdf3266f005507396fb0da2e12e6b22b662161070", size = 1810266, upload-time = "2025-07-07T14:19:51.565Z" },
+    { url = "https://files.pythonhosted.org/packages/55/9d/20a55786cc9d9266395544463d5db3be3528f7d5244bc52ba760de5dcc2d/dearpygui-2.1.0-cp312-cp312-macosx_10_6_x86_64.whl", hash = "sha256:1270ceb9cdb8ecc047c42477ccaa075b7864b314a5d09191f9280a24c8aa90a0", size = 2101499, upload-time = "2025-07-07T14:20:01.701Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/b2/39d820796b7ac4d0ebf93306c1f031bf3516b159408286f1fb495c6babeb/dearpygui-2.1.0-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:ce9969eb62057b9d4c88a8baaed13b5fbe4058caa9faf5b19fec89da75aece3d", size = 1874385, upload-time = "2025-07-07T14:20:11.226Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/26/c29998ffeb5eb8d638f307851e51a81c8bd4aeaf89ad660fc67ea4d1ac1a/dearpygui-2.1.0-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:a3ca8cf788db63ef7e2e8d6f277631b607d548b37606f080ca1b42b1f0a9b183", size = 2635863, upload-time = "2025-07-07T14:20:17.186Z" },
+    { url = "https://files.pythonhosted.org/packages/28/9c/3ab33927f1d8c839c5b7033a33d44fc9f0aeb00c264fc9772cb7555a03c4/dearpygui-2.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:43f0e4db9402f44fc3683a1f5c703564819de18cc15a042de7f1ed1c8cb5d148", size = 1810460, upload-time = "2025-07-07T14:19:53.13Z" },
+]
+
 [[package]]
 name = "dictdiffer"
 version = "0.9.0"
@@ -622,10 +637,10 @@ name = "gymnasium"
 version = "1.2.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "cloudpickle" },
-    { name = "farama-notifications" },
-    { name = "numpy" },
-    { name = "typing-extensions" },
+    { name = "cloudpickle", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "farama-notifications", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "numpy", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "typing-extensions", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/fd/17/c2a0e15c2cd5a8e788389b280996db927b923410de676ec5c7b2695e9261/gymnasium-1.2.0.tar.gz", hash = "sha256:344e87561012558f603880baf264ebc97f8a5c997a957b0c9f910281145534b0", size = 821142, upload-time = "2025-06-27T08:21:20.262Z" }
 wheels = [
@@ -903,22 +918,22 @@ name = "metadrive-simulator"
 version = "0.4.2.4"
 source = { url = "https://github.com/commaai/metadrive/releases/download/MetaDrive-minimal-0.4.2.4/metadrive_simulator-0.4.2.4-py3-none-any.whl" }
 dependencies = [
-    { name = "filelock" },
-    { name = "gymnasium" },
-    { name = "lxml" },
-    { name = "matplotlib" },
-    { name = "numpy" },
-    { name = "opencv-python-headless" },
-    { name = "panda3d" },
-    { name = "panda3d-gltf" },
-    { name = "pillow" },
-    { name = "progressbar" },
-    { name = "psutil" },
-    { name = "pygments" },
-    { name = "requests" },
-    { name = "shapely" },
-    { name = "tqdm" },
-    { name = "yapf" },
+    { name = "filelock", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "gymnasium", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "lxml", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "matplotlib", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "numpy", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "opencv-python-headless", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "panda3d", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "panda3d-gltf", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "pillow", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "progressbar", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "psutil", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "pygments", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "requests", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "shapely", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "tqdm", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "yapf", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
 ]
 wheels = [
     { url = "https://github.com/commaai/metadrive/releases/download/MetaDrive-minimal-0.4.2.4/metadrive_simulator-0.4.2.4-py3-none-any.whl", hash = "sha256:fbf0ea9be67e65cd45d38ff930e3d49f705dd76c9ddbd1e1482e3f87b61efcef" },
@@ -1237,6 +1252,7 @@ dependencies = [
     { name = "cffi" },
     { name = "crcmod" },
     { name = "cython" },
+    { name = "dearpygui" },
     { name = "future-fstrings" },
     { name = "inputs" },
     { name = "json-rpc" },
@@ -1327,6 +1343,7 @@ requires-dist = [
     { name = "crcmod" },
     { name = "cython" },
     { name = "dbus-next", marker = "extra == 'dev'" },
+    { name = "dearpygui", specifier = ">=2.1.0" },
     { name = "dictdiffer", marker = "extra == 'dev'" },
     { name = "future-fstrings" },
     { name = "hypothesis", marker = "extra == 'testing'", specifier = "==6.47.*" },
@@ -1422,8 +1439,8 @@ name = "panda3d-gltf"
 version = "0.13"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "panda3d" },
-    { name = "panda3d-simplepbr" },
+    { name = "panda3d", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "panda3d-simplepbr", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/07/7f/9f18fc3fa843a080acb891af6bcc12262e7bdf1d194a530f7042bebfc81f/panda3d-gltf-0.13.tar.gz", hash = "sha256:d06d373bdd91cf530909b669f43080e599463bbf6d3ef00c3558bad6c6b19675", size = 25573, upload-time = "2021-05-21T05:46:32.738Z" }
 wheels = [
@@ -1435,8 +1452,8 @@ name = "panda3d-simplepbr"
 version = "0.13.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "panda3d" },
-    { name = "typing-extensions" },
+    { name = "panda3d", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "typing-extensions", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/0d/be/c4d1ded04c22b357277cf6e6a44c1ab4abb285a700bd1991460460e05b99/panda3d_simplepbr-0.13.1.tar.gz", hash = "sha256:c83766d7c8f47499f365a07fe1dff078fc8b3054c2689bdc8dceabddfe7f1a35", size = 6216055, upload-time = "2025-03-30T16:57:41.087Z" }
 wheels = [
@@ -4173,9 +4190,9 @@ name = "pyopencl"
 version = "2025.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "numpy" },
-    { name = "platformdirs" },
-    { name = "pytools" },
+    { name = "numpy", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "platformdirs", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "pytools", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/28/88/0ac460d3e2def08b2ad6345db6a13613815f616bbbd60c6f4bdf774f4c41/pyopencl-2025.1.tar.gz", hash = "sha256:0116736d7f7920f87b8db4b66a03f27b1d930d2e37ddd14518407cc22dd24779", size = 422510, upload-time = "2025-01-22T00:16:58.421Z" }
 wheels = [
@@ -4351,7 +4368,7 @@ wheels = [
 
 [[package]]
 name = "pytest-xdist"
-version = "3.7.1.dev24+g2b4372bd6"
+version = "3.7.1.dev24+g2b4372b"
 source = { git = "https://github.com/sshane/pytest-xdist?rev=2b4372bd62699fb412c4fe2f95bf9f01bd2018da#2b4372bd62699fb412c4fe2f95bf9f01bd2018da" }
 dependencies = [
     { name = "execnet" },
@@ -4393,9 +4410,9 @@ name = "pytools"
 version = "2024.1.10"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "platformdirs" },
-    { name = "siphash24" },
-    { name = "typing-extensions" },
+    { name = "platformdirs", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "siphash24", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
+    { name = "typing-extensions", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/ee/0f/56e109c0307f831b5d598ad73976aaaa84b4d0e98da29a642e797eaa940c/pytools-2024.1.10.tar.gz", hash = "sha256:9af6f4b045212c49be32bb31fe19606c478ee4b09631886d05a32459f4ce0a12", size = 81741, upload-time = "2024-07-17T18:47:38.287Z" }
 wheels = [
@@ -4719,7 +4736,7 @@ name = "shapely"
 version = "2.1.1"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "numpy" },
+    { name = "numpy", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/ca/3c/2da625233f4e605155926566c0e7ea8dda361877f48e8b1655e53456f252/shapely-2.1.1.tar.gz", hash = "sha256:500621967f2ffe9642454808009044c21e5b35db89ce69f8a2042c2ffd0e2772", size = 315422, upload-time = "2025-05-19T11:04:41.265Z" }
 wheels = [
@@ -4948,7 +4965,7 @@ name = "yapf"
 version = "0.43.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
-    { name = "platformdirs" },
+    { name = "platformdirs", marker = "platform_machine != 'aarch64' or sys_platform != 'linux'" },
 ]
 sdist = { url = "https://files.pythonhosted.org/packages/23/97/b6f296d1e9cc1ec25c7604178b48532fa5901f721bcf1b8d8148b13e5588/yapf-0.43.0.tar.gz", hash = "sha256:00d3aa24bfedff9420b2e0d5d9f5ab6d9d4268e72afbf59bb3fa542781d5218e", size = 254907, upload-time = "2024-11-14T00:11:41.584Z" }
 wheels = [

From 5fb1e295141c7b05f9d50d653c956e935f4e33e8 Mon Sep 17 00:00:00 2001
From: "Quantizr (Jimmy)" <9859727+Quantizr@users.noreply.github.com>
Date: Fri, 22 Aug 2025 18:49:00 -0700
Subject: [PATCH 02/19] demo, executable, fontfile

---
 tools/jotpluggler/data.py    |  2 +-
 tools/jotpluggler/pluggle.py | 13 ++++++++++---
 2 files changed, 11 insertions(+), 4 deletions(-)
 mode change 100644 => 100755 tools/jotpluggler/pluggle.py

diff --git a/tools/jotpluggler/data.py b/tools/jotpluggler/data.py
index 2d15d23b7b..6381bea25a 100644
--- a/tools/jotpluggler/data.py
+++ b/tools/jotpluggler/data.py
@@ -85,7 +85,7 @@ class DataManager:
     self.time_series_data = {}
     self.loading = False
     self.route_start_time_mono = 0.0
-    self.duration = 100.0
+    self.duration = 0.0
     self._observers: list[Observer] = []
 
   def add_observer(self, observer: Observer):
diff --git a/tools/jotpluggler/pluggle.py b/tools/jotpluggler/pluggle.py
old mode 100644
new mode 100755
index 955009c083..32f40e55c4
--- a/tools/jotpluggler/pluggle.py
+++ b/tools/jotpluggler/pluggle.py
@@ -1,18 +1,23 @@
+#!/usr/bin/env python3
 import argparse
+import os
 import pyautogui
 import subprocess
 import dearpygui.dearpygui as dpg
 import threading
+from openpilot.common.basedir import BASEDIR
 from openpilot.tools.jotpluggler.data import DataManager, Observer, DataLoadedEvent
 from openpilot.tools.jotpluggler.views import DataTreeView
 from openpilot.tools.jotpluggler.layout import PlotLayoutManager, SplitterNode, LeafNode
 
+DEMO_ROUTE = "a2a0ccea32023010|2023-07-27--13-01-19"
+
 
 class PlaybackManager:
   def __init__(self):
     self.is_playing = False
     self.current_time_s = 0.0
-    self.duration_s = 100.0
+    self.duration_s = 0.0
     self.last_indices = {}
 
   def set_route_duration(self, duration: float):
@@ -206,7 +211,7 @@ def main(route_to_load=None):
     scale = 1
 
   with dpg.font_registry():
-    default_font = dpg.add_font("selfdrive/assets/fonts/Inter-Regular.ttf", int(13 * scale))
+    default_font = dpg.add_font(os.path.join(BASEDIR, "selfdrive/assets/fonts/Inter-Regular.ttf"), int(13 * scale))
   dpg.bind_font(default_font)
 
   viewport_width, viewport_height = int(1200 * scale), int(800 * scale)
@@ -235,6 +240,8 @@ def main(route_to_load=None):
 
 if __name__ == "__main__":
   parser = argparse.ArgumentParser(description="A tool for visualizing openpilot logs.")
+  parser.add_argument("--demo", action="store_true", help="Use the demo route instead of providing one")
   parser.add_argument("route", nargs='?', default=None, help="Optional route name to load on startup.")
   args = parser.parse_args()
-  main(route_to_load=args.route)
+  route = DEMO_ROUTE if args.demo else args.route
+  main(route_to_load=route)

From f4afe9c34761a83c5d3a24cb8b2f14c0c1121853 Mon Sep 17 00:00:00 2001
From: "Quantizr (Jimmy)" <9859727+Quantizr@users.noreply.github.com>
Date: Fri, 22 Aug 2025 19:14:17 -0700
Subject: [PATCH 03/19] calc max and min, numpy, cloudlog

---
 tools/jotpluggler/data.py | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/tools/jotpluggler/data.py b/tools/jotpluggler/data.py
index 6381bea25a..a9eca9766a 100644
--- a/tools/jotpluggler/data.py
+++ b/tools/jotpluggler/data.py
@@ -2,6 +2,7 @@ import threading
 import numpy as np
 from abc import ABC, abstractmethod
 from typing import Any
+from openpilot.common.swaglog import cloudlog
 from openpilot.tools.lib.logreader import LogReader
 from openpilot.tools.lib.log_time_series import msgs_to_time_series
 
@@ -28,12 +29,13 @@ class LogReaderSource(DataSource):
     raw_time_series = msgs_to_time_series(lr)
     processed_data = self._expand_list_fields(raw_time_series)
 
-    # Calculate timing information
-    times = [data['t'] for data in processed_data.values() if 't' in data and len(data['t']) > 0]
-    if times:
-      all_times = np.concatenate(times)
-      self._start_time_mono = all_times.min()
-      self._duration = all_times.max() - self._start_time_mono
+    min_time = float('inf')
+    max_time = float('-inf')
+    for data in processed_data.values():
+      min_time = min(min_time, data['t'][0])
+      max_time = max(max_time, data['t'][-1])
+    self._start_time_mono = min_time
+    self._duration = max_time - min_time
 
     return {'time_series_data': processed_data, 'route_start_time_mono': self._start_time_mono, 'duration': self._duration}
 
@@ -50,7 +52,7 @@ class LogReaderSource(DataSource):
           expanded_data[msg_type]['t'] = values
           continue
 
-        if isinstance(values, np.ndarray) and values.dtype == object:  # ragged array
+        if values.dtype == object:  # ragged array
           lens = np.fromiter((len(v) for v in values), dtype=int, count=len(values))
           max_len = lens.max() if lens.size else 0
           if max_len > 0:
@@ -60,7 +62,7 @@ class LogReaderSource(DataSource):
             for i in range(max_len):
               sub_arr = arr[:, i]
               expanded_data[msg_type][f"{field}/{i}"] = sub_arr
-        elif isinstance(values, np.ndarray) and values.ndim > 1:  # regular array
+        elif values.ndim > 1:  # regular multidimensional array
           for i in range(values.shape[1]):
             col_data = values[:, i]
             expanded_data[msg_type][f"{field}/{i}"] = col_data
@@ -173,7 +175,7 @@ class DataManager:
 
       self._notify_observers(DataLoadedEvent(data))
 
-    except Exception as e:
-      print(f"Error loading route: {e}")
+    except Exception:
+      cloudlog.exception("Error loading route:")
     finally:
       self.loading = False

From 2001898d8b11f2defe9a9455367ff535598f3db0 Mon Sep 17 00:00:00 2001
From: "Quantizr (Jimmy)" <9859727+Quantizr@users.noreply.github.com>
Date: Fri, 22 Aug 2025 21:05:33 -0700
Subject: [PATCH 04/19] mypy things

---
 tools/jotpluggler/layout.py  | 20 ++++++++------------
 tools/jotpluggler/pluggle.py |  2 +-
 tools/jotpluggler/views.py   | 25 +++++++++++--------------
 3 files changed, 20 insertions(+), 27 deletions(-)

diff --git a/tools/jotpluggler/layout.py b/tools/jotpluggler/layout.py
index 764477fbe8..0961635810 100644
--- a/tools/jotpluggler/layout.py
+++ b/tools/jotpluggler/layout.py
@@ -6,9 +6,9 @@ from openpilot.tools.jotpluggler.views import ViewPanel, TimeSeriesPanel
 
 
 class LayoutNode(ABC):
-  def __init__(self, node_id: str = None):
+  def __init__(self, node_id: str | None = None):
     self.node_id = node_id or str(uuid.uuid4())
-    self.tag = None
+    self.tag: str | None = None
 
   @abstractmethod
   def create_ui(self, parent_tag: str, width: int = -1, height: int = -1):
@@ -88,14 +88,12 @@ class LeafNode(LayoutNode):
 
 
 class SplitterNode(LayoutNode):
-  """Splitter node that contains multiple child nodes"""
-
-  def __init__(self, children: list[LayoutNode], orientation: str = "horizontal", node_id: str = None):
+  def __init__(self, children: list[LayoutNode], orientation: str = "horizontal", node_id: str | None = None):
     super().__init__(node_id)
     self.children = children if children else []
     self.orientation = orientation
     self.child_proportions = [1.0 / len(self.children) for _ in self.children] if self.children else []
-    self.child_container_tags = []  # Track container tags for resizing (different from child tag)
+    self.child_container_tags: list[str] = []  # Track container tags for resizing
 
   def preserve_data(self):
     for child in self.children:
@@ -194,7 +192,6 @@ class PlotLayoutManager:
     self.data_manager = data_manager
     self.playback_manager = playback_manager
     self.scale = scale
-    self.root_node: LayoutNode = None
     self.container_tag = "plot_layout_container"
     self._initialize_default_layout()
 
@@ -207,9 +204,8 @@ class PlotLayoutManager:
       dpg.delete_item(self.container_tag)
 
     with dpg.child_window(tag=self.container_tag, parent=parent_tag, border=False, width=-1, height=-1, no_scrollbar=True):
-      if self.root_node:
-        container_width, container_height = dpg.get_item_rect_size(self.container_tag)
-        self.root_node.create_ui(self.container_tag, container_width, container_height)
+      container_width, container_height = dpg.get_item_rect_size(self.container_tag)
+      self.root_node.create_ui(self.container_tag, container_width, container_height)
 
   def on_viewport_resize(self):
     if isinstance(self.root_node, SplitterNode):
@@ -290,8 +286,8 @@ class PlotLayoutManager:
           parent_width, parent_height = dpg.get_item_rect_size(parent_container)
           node.create_ui(parent_container, parent_width, parent_height)
 
-  def _find_parent_and_index(self, target_node: LayoutNode) -> tuple:  # TODO: probably can be stored in child
-    def search_recursive(node: LayoutNode, parent: LayoutNode = None, index: int = 0):
+  def _find_parent_and_index(self, target_node: LayoutNode):  # TODO: probably can be stored in child
+    def search_recursive(node: LayoutNode | None, parent: LayoutNode | None = None, index: int = 0):
       if node == target_node:
         return parent, index
       if isinstance(node, SplitterNode):
diff --git a/tools/jotpluggler/pluggle.py b/tools/jotpluggler/pluggle.py
index 32f40e55c4..c1feb6db0a 100755
--- a/tools/jotpluggler/pluggle.py
+++ b/tools/jotpluggler/pluggle.py
@@ -34,7 +34,7 @@ class PlaybackManager:
     self.current_time_s = max(0.0, min(time_s, self.duration_s))
     self.last_indices.clear()
 
-  def update_time(self, delta_t: float) -> float:
+  def update_time(self, delta_t: float):
     if self.is_playing:
       self.current_time_s = min(self.current_time_s + delta_t, self.duration_s)
       if self.current_time_s >= self.duration_s:
diff --git a/tools/jotpluggler/views.py b/tools/jotpluggler/views.py
index 163f60e647..0e86a18c44 100644
--- a/tools/jotpluggler/views.py
+++ b/tools/jotpluggler/views.py
@@ -32,21 +32,19 @@ class ViewPanel(ABC):
 
 
 class TimeSeriesPanel(ViewPanel, Observer):
-  def __init__(self, data_manager: DataManager, playback_manager, panel_id: str = None):
+  def __init__(self, data_manager: DataManager, playback_manager, panel_id: str | None = None):
     super().__init__(panel_id)
     self.data_manager = data_manager
     self.playback_manager = playback_manager
     self.title = "Time Series Plot"
     self.plotted_series: set[str] = set()
-    self.plot_tag = None
-    self.x_axis_tag = None
-    self.y_axis_tag = None
-    self.timeline_indicator_tag = None
+    self.plot_tag: str | None = None
+    self.x_axis_tag: str | None = None
+    self.y_axis_tag: str | None = None
+    self.timeline_indicator_tag: str | None = None
     self._ui_created = False
-
-    # Store series data for restoration and legend management
-    self._preserved_series_data = []  # TODO: the way we do this right now doesn't make much sense
-    self._series_legend_tags = {}  # Maps series_path to legend tag
+    self._preserved_series_data: list[tuple[str, tuple]] = []  # TODO: the way we do this right now doesn't make much sense
+    self._series_legend_tags: dict[str, str] = {}  # Maps series_path to legend tag
 
     self.data_manager.add_observer(self)
 
@@ -110,7 +108,7 @@ class TimeSeriesPanel(ViewPanel, Observer):
           if dpg.does_item_exist(series_tag):
             dpg.configure_item(series_tag, label=legend_label)
 
-  def _add_series_with_data(self, series_path: str, rel_time_array, value_array):
+  def _add_series_with_data(self, series_path: str, rel_time_array, value_array) -> bool:
     if series_path in self.plotted_series:
       return False
 
@@ -163,7 +161,7 @@ class TimeSeriesPanel(ViewPanel, Observer):
     for series_path in self.plotted_series.copy():
       self._update_series_data(series_path)
 
-  def _update_series_data(self, series_path: str):
+  def _update_series_data(self, series_path: str) -> bool:
     time_value_data = self.data_manager.get_time_series_data(series_path)
     if time_value_data is None:
       return False
@@ -189,17 +187,16 @@ class DataTreeNode:
   def __init__(self, name: str, full_path: str = ""):
     self.name = name
     self.full_path = full_path
-    self.children = {}
+    self.children: dict[str, DataTreeNode] = {}
     self.is_leaf = False
 
-
 class DataTreeView(Observer):
   def __init__(self, data_manager: DataManager, ui_lock: threading.Lock):
     self.data_manager = data_manager
     self.ui_lock = ui_lock
     self.current_search = ""
     self.data_tree = DataTreeNode(name="root")
-    self.active_leaf_nodes = []
+    self.active_leaf_nodes: list[DataTreeNode] = []
     self.data_manager.add_observer(self)
 
   def on_data_loaded(self, event: DataLoadedEvent):

From 6048eb568aa57153affc69e7c74c1975a694cc8b Mon Sep 17 00:00:00 2001
From: "Quantizr (Jimmy)" <9859727+Quantizr@users.noreply.github.com>
Date: Sat, 23 Aug 2025 03:15:15 -0700
Subject: [PATCH 05/19] simplified data.py

---
 tools/jotpluggler/data.py    | 167 ++++++++++++++---------------------
 tools/jotpluggler/pluggle.py |  16 ++--
 tools/jotpluggler/views.py   |  25 +++---
 3 files changed, 85 insertions(+), 123 deletions(-)

diff --git a/tools/jotpluggler/data.py b/tools/jotpluggler/data.py
index a9eca9766a..2b31df1811 100644
--- a/tools/jotpluggler/data.py
+++ b/tools/jotpluggler/data.py
@@ -1,107 +1,34 @@
 import threading
 import numpy as np
-from abc import ABC, abstractmethod
-from typing import Any
+from collections.abc import Callable
 from openpilot.common.swaglog import cloudlog
 from openpilot.tools.lib.logreader import LogReader
 from openpilot.tools.lib.log_time_series import msgs_to_time_series
 
 
-# TODO: support cereal/ZMQ streaming
-class DataSource(ABC):
-  @abstractmethod
-  def load_data(self) -> dict[str, Any]:
-    pass
-
-  @abstractmethod
-  def get_duration(self) -> float:
-    pass
-
-
-class LogReaderSource(DataSource):
-  def __init__(self, route_name: str):
-    self.route_name = route_name
-    self._duration = 0.0
-    self._start_time_mono = 0.0
-
-  def load_data(self) -> dict[str, Any]:
-    lr = LogReader(self.route_name)
-    raw_time_series = msgs_to_time_series(lr)
-    processed_data = self._expand_list_fields(raw_time_series)
-
-    min_time = float('inf')
-    max_time = float('-inf')
-    for data in processed_data.values():
-      min_time = min(min_time, data['t'][0])
-      max_time = max(max_time, data['t'][-1])
-    self._start_time_mono = min_time
-    self._duration = max_time - min_time
-
-    return {'time_series_data': processed_data, 'route_start_time_mono': self._start_time_mono, 'duration': self._duration}
-
-  def get_duration(self) -> float:
-    return self._duration
-
-  # TODO: lists are expanded, but lists of structs are not
-  def _expand_list_fields(self, time_series_data):
-    expanded_data = {}
-    for msg_type, data in time_series_data.items():
-      expanded_data[msg_type] = {}
-      for field, values in data.items():
-        if field == 't':
-          expanded_data[msg_type]['t'] = values
-          continue
-
-        if values.dtype == object:  # ragged array
-          lens = np.fromiter((len(v) for v in values), dtype=int, count=len(values))
-          max_len = lens.max() if lens.size else 0
-          if max_len > 0:
-            arr = np.full((len(values), max_len), None, dtype=object)
-            for i, v in enumerate(values):
-              arr[i, : lens[i]] = v
-            for i in range(max_len):
-              sub_arr = arr[:, i]
-              expanded_data[msg_type][f"{field}/{i}"] = sub_arr
-        elif values.ndim > 1:  # regular multidimensional array
-          for i in range(values.shape[1]):
-            col_data = values[:, i]
-            expanded_data[msg_type][f"{field}/{i}"] = col_data
-        else:
-          expanded_data[msg_type][field] = values
-    return expanded_data
-
-
-class DataLoadedEvent:
-  def __init__(self, data: dict[str, Any]):
-    self.data = data
-
-
-class Observer(ABC):
-  @abstractmethod
-  def on_data_loaded(self, event: DataLoadedEvent):
-    pass
-
-
 class DataManager:
   def __init__(self):
     self.time_series_data = {}
     self.loading = False
     self.route_start_time_mono = 0.0
     self.duration = 0.0
-    self._observers: list[Observer] = []
+    self._callbacks: list[Callable[[dict], None]] = []
 
-  def add_observer(self, observer: Observer):
-    self._observers.append(observer)
+  def add_callback(self, callback: Callable[[dict], None]):
+    self._callbacks.append(callback)
 
-  def remove_observer(self, observer: Observer):
-    if observer in self._observers:
-      self._observers.remove(observer)
+  def remove_callback(self, callback: Callable[[dict], None]):
+    if callback in self._callbacks:
+      self._callbacks.remove(callback)
 
-  def _notify_observers(self, event: DataLoadedEvent):
-    for observer in self._observers:
-      observer.on_data_loaded(event)
+  def _notify_callbacks(self, data: dict):
+    for callback in self._callbacks:
+      try:
+        callback(data)
+      except Exception as e:
+        cloudlog.exception(f"Error in data callback: {e}")
 
-  def get_current_value_for_path(self, path: str, time_s: float, last_index: int | None = None):
+  def get_current_value(self, path: str, time_s: float, last_index: int | None = None):
     try:
       abs_time_s = self.route_start_time_mono + time_s
       msg_type, field_path = path.split('/', 1)
@@ -136,24 +63,24 @@ class DataManager:
     try:
       msg_type, field_path = path.split('/', 1)
       value_array = self.time_series_data.get(msg_type, {}).get(field_path)
-      if value_array is not None:  # only numbers and bools are plottable
+      if value_array is not None:
         return np.issubdtype(value_array.dtype, np.number) or np.issubdtype(value_array.dtype, np.bool_)
     except (ValueError, KeyError):
       pass
     return False
 
-  def get_time_series_data(self, path: str) -> tuple | None:
+  def get_time_series(self, path: str):
     try:
       msg_type, field_path = path.split('/', 1)
       ts_data = self.time_series_data[msg_type]
       time_array = ts_data['t']
-      plot_values = ts_data[field_path]
+      values = ts_data[field_path]
 
       if len(time_array) == 0:
         return None
 
-      rel_time_array = time_array - self.route_start_time_mono
-      return rel_time_array, plot_values
+      rel_time = time_array - self.route_start_time_mono
+      return rel_time, values
 
     except (KeyError, ValueError):
       return None
@@ -163,19 +90,55 @@ class DataManager:
       return
 
     self.loading = True
-    data_source = LogReaderSource(route_name)
-    threading.Thread(target=self._load_in_background, args=(data_source,), daemon=True).start()
+    threading.Thread(target=self._load_route_background, args=(route_name,), daemon=True).start()
 
-  def _load_in_background(self, data_source: DataSource):
+  def _load_route_background(self, route_name: str):
     try:
-      data = data_source.load_data()
-      self.time_series_data = data['time_series_data']
-      self.route_start_time_mono = data['route_start_time_mono']
-      self.duration = data['duration']
+      lr = LogReader(route_name)
+      raw_data = msgs_to_time_series(lr)
+      processed_data = self._expand_list_fields(raw_data)
 
-      self._notify_observers(DataLoadedEvent(data))
+      min_time = float('inf')
+      max_time = float('-inf')
+      for data in processed_data.values():
+        if len(data['t']) > 0:
+          min_time = min(min_time, data['t'][0])
+          max_time = max(max_time, data['t'][-1])
 
-    except Exception:
-      cloudlog.exception("Error loading route:")
+      self.time_series_data = processed_data
+      self.route_start_time_mono = min_time if min_time != float('inf') else 0.0
+      self.duration = max_time - min_time if max_time != float('-inf') else 0.0
+
+      self._notify_callbacks({'time_series_data': processed_data, 'route_start_time_mono': self.route_start_time_mono, 'duration': self.duration})
+
+    except Exception as e:
+      cloudlog.exception(f"Error loading route {route_name}: {e}")
     finally:
       self.loading = False
+
+  def _expand_list_fields(self, time_series_data):
+    expanded_data = {}
+    for msg_type, data in time_series_data.items():
+      expanded_data[msg_type] = {}
+      for field, values in data.items():
+        if field == 't':
+          expanded_data[msg_type]['t'] = values
+          continue
+
+        if values.dtype == object:  # ragged array
+          lens = np.fromiter((len(v) for v in values), dtype=int, count=len(values))
+          max_len = lens.max() if lens.size else 0
+          if max_len > 0:
+            arr = np.full((len(values), max_len), None, dtype=object)
+            for i, v in enumerate(values):
+              arr[i, : lens[i]] = v
+            for i in range(max_len):
+              sub_arr = arr[:, i]
+              expanded_data[msg_type][f"{field}/{i}"] = sub_arr
+        elif values.ndim > 1:  # regular multidimensional array
+          for i in range(values.shape[1]):
+            col_data = values[:, i]
+            expanded_data[msg_type][f"{field}/{i}"] = col_data
+        else:
+          expanded_data[msg_type][field] = values
+    return expanded_data
diff --git a/tools/jotpluggler/pluggle.py b/tools/jotpluggler/pluggle.py
index c1feb6db0a..e2c9df159b 100755
--- a/tools/jotpluggler/pluggle.py
+++ b/tools/jotpluggler/pluggle.py
@@ -6,7 +6,7 @@ import subprocess
 import dearpygui.dearpygui as dpg
 import threading
 from openpilot.common.basedir import BASEDIR
-from openpilot.tools.jotpluggler.data import DataManager, Observer, DataLoadedEvent
+from openpilot.tools.jotpluggler.data import DataManager
 from openpilot.tools.jotpluggler.views import DataTreeView
 from openpilot.tools.jotpluggler.layout import PlotLayoutManager, SplitterNode, LeafNode
 
@@ -61,7 +61,7 @@ def format_and_truncate(value, available_width: float, avg_char_width: float) ->
   return s
 
 
-class MainController(Observer):
+class MainController:
   def __init__(self, scale: float = 1.0):
     self.ui_lock = threading.Lock()
     self.scale = scale
@@ -70,7 +70,7 @@ class MainController(Observer):
     self._create_global_themes()
     self.data_tree_view = DataTreeView(self.data_manager, self.ui_lock)
     self.plot_layout_manager = PlotLayoutManager(self.data_manager, self.playback_manager, scale=self.scale)
-    self.data_manager.add_observer(self)
+    self.data_manager.add_callback(self.on_data_loaded)
     self.avg_char_width = None
 
   def _create_global_themes(self):
@@ -85,12 +85,12 @@ class MainController(Observer):
         dpg.add_theme_style(dpg.mvPlotStyleVar_LineWeight, scaled_thickness, category=dpg.mvThemeCat_Plots)
         dpg.add_theme_color(dpg.mvPlotCol_Line, (255, 0, 0, 128), category=dpg.mvThemeCat_Plots)
 
-  def on_data_loaded(self, event: DataLoadedEvent):
-    self.playback_manager.set_route_duration(event.data['duration'])
-    num_msg_types = len(event.data['time_series_data'])
+  def on_data_loaded(self, data: dict):
+    self.playback_manager.set_route_duration(data['duration'])
+    num_msg_types = len(data['time_series_data'])
     dpg.set_value("load_status", f"Loaded {num_msg_types} message types")
     dpg.configure_item("load_button", enabled=True)
-    dpg.configure_item("timeline_slider", max_value=event.data['duration'])
+    dpg.configure_item("timeline_slider", max_value=data['duration'])
 
   def setup_ui(self):
     with dpg.item_handler_registry(tag="tree_node_handler"):
@@ -180,7 +180,7 @@ class MainController(Observer):
 
       if dpg.does_item_exist(value_tag) and dpg.is_item_visible(value_tag):
         last_index = self.playback_manager.last_indices.get(path)
-        value, new_idx = self.data_manager.get_current_value_for_path(path, self.playback_manager.current_time_s, last_index)
+        value, new_idx = self.data_manager.get_current_value(path, self.playback_manager.current_time_s, last_index)
 
         if value is not None:
           self.playback_manager.update_index(path, new_idx)
diff --git a/tools/jotpluggler/views.py b/tools/jotpluggler/views.py
index 0e86a18c44..feeff7ab24 100644
--- a/tools/jotpluggler/views.py
+++ b/tools/jotpluggler/views.py
@@ -4,7 +4,7 @@ import uuid
 import threading
 import dearpygui.dearpygui as dpg
 from abc import ABC, abstractmethod
-from openpilot.tools.jotpluggler.data import Observer, DataLoadedEvent, DataManager
+from openpilot.tools.jotpluggler.data import DataManager
 
 
 class ViewPanel(ABC):
@@ -31,7 +31,7 @@ class ViewPanel(ABC):
     pass
 
 
-class TimeSeriesPanel(ViewPanel, Observer):
+class TimeSeriesPanel(ViewPanel):
   def __init__(self, data_manager: DataManager, playback_manager, panel_id: str | None = None):
     super().__init__(panel_id)
     self.data_manager = data_manager
@@ -45,14 +45,13 @@ class TimeSeriesPanel(ViewPanel, Observer):
     self._ui_created = False
     self._preserved_series_data: list[tuple[str, tuple]] = []  # TODO: the way we do this right now doesn't make much sense
     self._series_legend_tags: dict[str, str] = {}  # Maps series_path to legend tag
-
-    self.data_manager.add_observer(self)
+    self.data_manager.add_callback(self.on_data_loaded)
 
   def preserve_data(self):
     self._preserved_series_data = []
     if self.plotted_series and self._ui_created:
       for series_path in self.plotted_series:
-        time_value_data = self.data_manager.get_time_series_data(series_path)
+        time_value_data = self.data_manager.get_time_series(series_path)
         if time_value_data:
           self._preserved_series_data.append((series_path, time_value_data))
 
@@ -88,7 +87,7 @@ class TimeSeriesPanel(ViewPanel, Observer):
     if self.plotted_series:  # update legend labels with current values
       for series_path in self.plotted_series:
         last_index = self.playback_manager.last_indices.get(series_path)
-        value, new_idx = self.data_manager.get_current_value_for_path(series_path, current_time_s, last_index)
+        value, new_idx = self.data_manager.get_current_value(series_path, current_time_s, last_index)
 
         if value is not None:
           self.playback_manager.update_index(series_path, new_idx)
@@ -126,7 +125,7 @@ class TimeSeriesPanel(ViewPanel, Observer):
     if self.plot_tag and dpg.does_item_exist(self.plot_tag):
       dpg.delete_item(self.plot_tag)
 
-    # self.data_manager.remove_observer(self)
+    # self.data_manager.remove_callback(self.on_data_loaded)
     self._series_legend_tags.clear()
     self._ui_created = False
 
@@ -137,7 +136,7 @@ class TimeSeriesPanel(ViewPanel, Observer):
     if series_path in self.plotted_series:
       return False
 
-    time_value_data = self.data_manager.get_time_series_data(series_path)
+    time_value_data = self.data_manager.get_time_series(series_path)
     if time_value_data is None:
       return False
 
@@ -157,12 +156,12 @@ class TimeSeriesPanel(ViewPanel, Observer):
       if series_path in self._series_legend_tags:  # Clean up legend tag mapping
         del self._series_legend_tags[series_path]
 
-  def on_data_loaded(self, event: DataLoadedEvent):
+  def on_data_loaded(self, data: dict):
     for series_path in self.plotted_series.copy():
       self._update_series_data(series_path)
 
   def _update_series_data(self, series_path: str) -> bool:
-    time_value_data = self.data_manager.get_time_series_data(series_path)
+    time_value_data = self.data_manager.get_time_series(series_path)
     if time_value_data is None:
       return False
 
@@ -190,16 +189,16 @@ class DataTreeNode:
     self.children: dict[str, DataTreeNode] = {}
     self.is_leaf = False
 
-class DataTreeView(Observer):
+class DataTreeView:
   def __init__(self, data_manager: DataManager, ui_lock: threading.Lock):
     self.data_manager = data_manager
     self.ui_lock = ui_lock
     self.current_search = ""
     self.data_tree = DataTreeNode(name="root")
     self.active_leaf_nodes: list[DataTreeNode] = []
-    self.data_manager.add_observer(self)
+    self.data_manager.add_callback(self.on_data_loaded)
 
-  def on_data_loaded(self, event: DataLoadedEvent):
+  def on_data_loaded(self, data: dict):
     with self.ui_lock:
       self.populate_data_tree()
 

From 2eb697a730bf05f5ccb5f7e57c6a82b5692a832e Mon Sep 17 00:00:00 2001
From: "Quantizr (Jimmy)" <9859727+Quantizr@users.noreply.github.com>
Date: Wed, 27 Aug 2025 00:45:14 -0700
Subject: [PATCH 06/19] multiprocessed data ingest

---
 tools/jotpluggler/data.py    | 393 ++++++++++++++++++++++++-----------
 tools/jotpluggler/pluggle.py |  30 ++-
 tools/jotpluggler/views.py   |  25 +--
 3 files changed, 297 insertions(+), 151 deletions(-)

diff --git a/tools/jotpluggler/data.py b/tools/jotpluggler/data.py
index 2b31df1811..57b6ac972c 100644
--- a/tools/jotpluggler/data.py
+++ b/tools/jotpluggler/data.py
@@ -1,144 +1,301 @@
-import threading
 import numpy as np
-from collections.abc import Callable
+import threading
+import multiprocessing
+import bisect
+from collections import defaultdict
+from typing import Any
+import tqdm
 from openpilot.common.swaglog import cloudlog
-from openpilot.tools.lib.logreader import LogReader
-from openpilot.tools.lib.log_time_series import msgs_to_time_series
+from openpilot.tools.lib.logreader import _LogFileReader, LogReader
+
+
+def flatten_dict(d: dict, sep: str = "/", prefix: str = None) -> dict:
+  result = {}
+  stack = [(d, prefix)]
+
+  while stack:
+    obj, current_prefix = stack.pop()
+
+    if isinstance(obj, dict):
+      for key, val in obj.items():
+        new_prefix = key if current_prefix is None else f"{current_prefix}{sep}{key}"
+        if isinstance(val, (dict, list)):
+          stack.append((val, new_prefix))
+        else:
+          result[new_prefix] = val
+    elif isinstance(obj, list):
+      for i, item in enumerate(obj):
+        new_prefix = f"{current_prefix}{sep}{i}"
+        if isinstance(item, (dict, list)):
+          stack.append((item, new_prefix))
+        else:
+          result[new_prefix] = item
+    else:
+      if current_prefix is not None:
+        result[current_prefix] = obj
+  return result
+
+
+def extract_field_types(schema, prefix, field_types_dict):
+  stack = [(schema, prefix)]
+
+  while stack:
+    current_schema, current_prefix = stack.pop()
+
+    for field in current_schema.fields_list:
+      field_name = field.proto.name
+      field_path = f"{current_prefix}/{field_name}"
+      field_proto = field.proto
+      field_which = field_proto.which()
+
+      field_type = field_proto.slot.type.which() if field_which == 'slot' else field_which
+      field_types_dict[field_path] = field_type
+
+      if field_which == 'slot':
+        slot_type = field_proto.slot.type
+        type_which = slot_type.which()
+
+        if type_which == 'list':
+          element_type = slot_type.list.elementType.which()
+          list_path = f"{field_path}/*"
+          field_types_dict[list_path] = element_type
+
+          if element_type == 'struct':
+            stack.append((field.schema.elementType, list_path))
+
+        elif type_which == 'struct':
+          stack.append((field.schema, field_path))
+
+      elif field_which == 'group':
+        stack.append((field.schema, field_path))
+
+
+def _convert_to_optimal_dtype(values_list, capnp_type):
+  if not values_list:
+    return np.array([])
+
+  dtype_mapping = {
+    'bool': np.bool_, 'int8': np.int8, 'int16': np.int16, 'int32': np.int32, 'int64': np.int64,
+    'uint8': np.uint8, 'uint16': np.uint16, 'uint32': np.uint32, 'uint64': np.uint64,
+    'float32': np.float32, 'float64': np.float64, 'text': object, 'data': object,
+    'enum': object, 'anyPointer': object,
+  }
+
+  target_dtype = dtype_mapping.get(capnp_type)
+  return np.array(values_list, dtype=target_dtype) if target_dtype else np.array(values_list)
+
+
+def _match_field_type(field_path, field_types):
+  if field_path in field_types:
+    return field_types[field_path]
+
+  path_parts = field_path.split('/')
+  template_parts = [p if not p.isdigit() else '*' for p in path_parts]
+  template_path = '/'.join(template_parts)
+  return field_types.get(template_path)
+
+
+def msgs_to_time_series(msgs):
+  """Extract scalar fields and return (time_series_data, start_time, end_time)."""
+  collected_data = defaultdict(lambda: {'timestamps': [], 'columns': defaultdict(list), 'sparse_fields': set()})
+  field_types = {}
+  extracted_schemas = set()
+  min_time = max_time = None
+
+  for msg in msgs:
+    typ = msg.which()
+    timestamp = msg.logMonoTime * 1e-9
+    if typ != 'initData':
+      if min_time is None:
+        min_time = timestamp
+      max_time = timestamp
+
+    sub_msg = getattr(msg, typ)
+    if not hasattr(sub_msg, 'to_dict') or typ in ('qcomGnss', 'ubloxGnss'):
+      continue
+
+    if hasattr(sub_msg, 'schema') and typ not in extracted_schemas:
+      extract_field_types(sub_msg.schema, typ, field_types)
+      extracted_schemas.add(typ)
+
+    msg_dict = sub_msg.to_dict(verbose=True)
+    flat_dict = flatten_dict(msg_dict)
+    flat_dict['_valid'] = msg.valid
+
+    type_data = collected_data[typ]
+    columns, sparse_fields = type_data['columns'], type_data['sparse_fields']
+    known_fields = set(columns.keys())
+    missing_fields = known_fields - flat_dict.keys()
+
+    for field, value in flat_dict.items():
+      if field not in known_fields and type_data['timestamps']:
+        sparse_fields.add(field)
+      columns[field].append(value)
+      if value is None:
+        sparse_fields.add(field)
+
+    for field in missing_fields:
+      columns[field].append(None)
+      sparse_fields.add(field)
+
+    type_data['timestamps'].append(timestamp)
+
+  final_result = {}
+  for typ, data in collected_data.items():
+    if not data['timestamps']:
+      continue
+
+    typ_result = {'t': np.array(data['timestamps'], dtype=np.float64)}
+    sparse_fields = data['sparse_fields']
+
+    for field_name, values in data['columns'].items():
+      if len(values) < len(data['timestamps']):
+        values = [None] * (len(data['timestamps']) - len(values)) + values
+        sparse_fields.add(field_name)
+
+      if field_name in sparse_fields:
+        typ_result[field_name] = np.array(values, dtype=object)
+      else:
+        capnp_type = _match_field_type(f"{typ}/{field_name}", field_types)
+        typ_result[field_name] = _convert_to_optimal_dtype(values, capnp_type)
+
+    final_result[typ] = typ_result
+
+  return final_result, min_time or 0.0, max_time or 0.0
+
+
+def _process_segment(segment_identifier: str) -> tuple[dict[str, Any], float, float]:
+  try:
+    lr = _LogFileReader(segment_identifier, sort_by_time=True)
+    return msgs_to_time_series(lr)
+  except Exception as e:
+    cloudlog.warning(f"Warning: Failed to process segment {segment_identifier}: {e}")
+    return {}, 0.0, 0.0
 
 
 class DataManager:
   def __init__(self):
-    self.time_series_data = {}
+    self._segments = []
+    self._segment_starts = []
+    self._start_time = 0.0
+    self._duration = 0.0
+    self._paths = set()
+    self._observers = []
     self.loading = False
-    self.route_start_time_mono = 0.0
-    self.duration = 0.0
-    self._callbacks: list[Callable[[dict], None]] = []
-
-  def add_callback(self, callback: Callable[[dict], None]):
-    self._callbacks.append(callback)
+    self._lock = threading.RLock()
 
-  def remove_callback(self, callback: Callable[[dict], None]):
-    if callback in self._callbacks:
-      self._callbacks.remove(callback)
+  def load_route(self, route: str) -> None:
+    if self.loading:
+      return
+    self._reset()
+    threading.Thread(target=self._load_async, args=(route,), daemon=True).start()
 
-  def _notify_callbacks(self, data: dict):
-    for callback in self._callbacks:
-      try:
-        callback(data)
-      except Exception as e:
-        cloudlog.exception(f"Error in data callback: {e}")
+  def get_timeseries(self, path: str):
+    with self._lock:
+      msg_type, field = path.split('/', 1)
+      times, values = [], []
 
-  def get_current_value(self, path: str, time_s: float, last_index: int | None = None):
-    try:
-      abs_time_s = self.route_start_time_mono + time_s
-      msg_type, field_path = path.split('/', 1)
-      ts_data = self.time_series_data[msg_type]
-      t, v = ts_data['t'], ts_data[field_path]
-
-      if len(t) == 0:
-        return None, None
-
-      if last_index is None:  # jump
-        idx = np.searchsorted(t, abs_time_s, side='right') - 1
-      else:  # continuous playback
-        idx = last_index
-        while idx < len(t) - 1 and t[idx + 1] < abs_time_s:
-          idx += 1
-
-      idx = max(0, idx)
-      return v[idx], idx
-
-    except (KeyError, IndexError):
-      return None, None
-
-  def get_all_paths(self) -> list[str]:
-    all_paths = []
-    for msg_type, data in self.time_series_data.items():
-      for key in data.keys():
-        if key != 't':
-          all_paths.append(f"{msg_type}/{key}")
-    return all_paths
-
-  def is_path_plottable(self, path: str) -> bool:
-    try:
-      msg_type, field_path = path.split('/', 1)
-      value_array = self.time_series_data.get(msg_type, {}).get(field_path)
-      if value_array is not None:
-        return np.issubdtype(value_array.dtype, np.number) or np.issubdtype(value_array.dtype, np.bool_)
-    except (ValueError, KeyError):
-      pass
-    return False
-
-  def get_time_series(self, path: str):
-    try:
-      msg_type, field_path = path.split('/', 1)
-      ts_data = self.time_series_data[msg_type]
-      time_array = ts_data['t']
-      values = ts_data[field_path]
+      for segment in self._segments:
+        if msg_type in segment and field in segment[msg_type]:
+          times.append(segment[msg_type]['t'])
+          values.append(segment[msg_type][field])
 
-      if len(time_array) == 0:
+      if not times:
         return None
 
-      rel_time = time_array - self.route_start_time_mono
-      return rel_time, values
+      combined_times = np.concatenate(times) - self._start_time
+      if len(values) > 1 and any(arr.dtype != values[0].dtype for arr in values):
+        values = [arr.astype(object) for arr in values]
+
+      return combined_times, np.concatenate(values)
 
-    except (KeyError, ValueError):
+  def get_value_at(self, path: str, time: float):
+    with self._lock:
+      absolute_time = self._start_time + time
+      message_type, field = path.split('/', 1)
+      current_index = bisect.bisect_right(self._segment_starts, absolute_time) - 1
+      for index in (current_index, current_index - 1):
+        if not 0 <= index < len(self._segments):
+          continue
+        segment = self._segments[index].get(message_type)
+        if not segment or field not in segment:
+          continue
+        times = segment['t']
+        if len(times) == 0 or (index != current_index and absolute_time - times[-1] > 1):
+          continue
+        position = np.searchsorted(times, absolute_time, 'right') - 1
+        if position >= 0 and absolute_time - times[position] <= 1:
+          return segment[field][position]
       return None
 
-  def load_route(self, route_name: str):
-    if self.loading:
-      return
+  def get_all_paths(self):
+    with self._lock:
+      return sorted(self._paths)
 
-    self.loading = True
-    threading.Thread(target=self._load_route_background, args=(route_name,), daemon=True).start()
+  def get_duration(self):
+    with self._lock:
+      return self._duration
 
-  def _load_route_background(self, route_name: str):
-    try:
-      lr = LogReader(route_name)
-      raw_data = msgs_to_time_series(lr)
-      processed_data = self._expand_list_fields(raw_data)
+  def is_plottable(self, path: str):
+    data = self.get_timeseries(path)
+    if data is None:
+      return False
+    _, values = data
+    return np.issubdtype(values.dtype, np.number) or np.issubdtype(values.dtype, np.bool_)
 
-      min_time = float('inf')
-      max_time = float('-inf')
-      for data in processed_data.values():
-        if len(data['t']) > 0:
-          min_time = min(min_time, data['t'][0])
-          max_time = max(max_time, data['t'][-1])
+  def add_observer(self, callback):
+    with self._lock:
+      self._observers.append(callback)
 
-      self.time_series_data = processed_data
-      self.route_start_time_mono = min_time if min_time != float('inf') else 0.0
-      self.duration = max_time - min_time if max_time != float('-inf') else 0.0
+  def _reset(self):
+    with self._lock:
+      self.loading = True
+      self._segments.clear()
+      self._segment_starts.clear()
+      self._paths.clear()
+      self._start_time = self._duration = 0.0
 
-      self._notify_callbacks({'time_series_data': processed_data, 'route_start_time_mono': self.route_start_time_mono, 'duration': self.duration})
+  def _load_async(self, route: str):
+    try:
+      lr = LogReader(route, sort_by_time=True)
+      if not lr.logreader_identifiers:
+        cloudlog.warning(f"Warning: No log segments found for route: {route}")
+        return
 
+      with multiprocessing.Pool() as pool, tqdm.tqdm(total=len(lr.logreader_identifiers), desc="Processing Segments") as pbar:
+        for segment_result, start_time, end_time in pool.imap(_process_segment, lr.logreader_identifiers):
+          pbar.update(1)
+          if segment_result:
+            self._add_segment(segment_result, start_time, end_time)
     except Exception as e:
-      cloudlog.exception(f"Error loading route {route_name}: {e}")
+      cloudlog.exception(f"Error loading route {route}:")
     finally:
-      self.loading = False
+      self._finalize_loading()
 
-  def _expand_list_fields(self, time_series_data):
-    expanded_data = {}
-    for msg_type, data in time_series_data.items():
-      expanded_data[msg_type] = {}
-      for field, values in data.items():
-        if field == 't':
-          expanded_data[msg_type]['t'] = values
-          continue
+  def _add_segment(self, segment_data: dict, start_time: float, end_time: float):
+    with self._lock:
+      self._segments.append(segment_data)
+      self._segment_starts.append(start_time)
 
-        if values.dtype == object:  # ragged array
-          lens = np.fromiter((len(v) for v in values), dtype=int, count=len(values))
-          max_len = lens.max() if lens.size else 0
-          if max_len > 0:
-            arr = np.full((len(values), max_len), None, dtype=object)
-            for i, v in enumerate(values):
-              arr[i, : lens[i]] = v
-            for i in range(max_len):
-              sub_arr = arr[:, i]
-              expanded_data[msg_type][f"{field}/{i}"] = sub_arr
-        elif values.ndim > 1:  # regular multidimensional array
-          for i in range(values.shape[1]):
-            col_data = values[:, i]
-            expanded_data[msg_type][f"{field}/{i}"] = col_data
-        else:
-          expanded_data[msg_type][field] = values
-    return expanded_data
+      if len(self._segments) == 1:
+        self._start_time = start_time
+      self._duration = end_time - self._start_time
+
+      for msg_type, data in segment_data.items():
+        for field in data.keys():
+          if field != 't':
+            self._paths.add(f"{msg_type}/{field}")
+
+      observers = self._observers.copy()
+
+    for callback in observers:
+      callback({'segment_added': True, 'duration': self._duration})
+
+  def _finalize_loading(self):
+    with self._lock:
+      self.loading = False
+      observers = self._observers.copy()
+      duration = self._duration
+
+    for callback in observers:
+      callback({'loading_complete': True, 'duration': duration})
diff --git a/tools/jotpluggler/pluggle.py b/tools/jotpluggler/pluggle.py
index e2c9df159b..614bb53e8e 100755
--- a/tools/jotpluggler/pluggle.py
+++ b/tools/jotpluggler/pluggle.py
@@ -18,7 +18,6 @@ class PlaybackManager:
     self.is_playing = False
     self.current_time_s = 0.0
     self.duration_s = 0.0
-    self.last_indices = {}
 
   def set_route_duration(self, duration: float):
     self.duration_s = duration
@@ -32,7 +31,6 @@ class PlaybackManager:
   def seek(self, time_s: float):
     self.is_playing = False
     self.current_time_s = max(0.0, min(time_s, self.duration_s))
-    self.last_indices.clear()
 
   def update_time(self, delta_t: float):
     if self.is_playing:
@@ -41,10 +39,6 @@ class PlaybackManager:
         self.is_playing = False
     return self.current_time_s
 
-  def update_index(self, path: str, new_idx: int | None):
-    if new_idx is not None:
-      self.last_indices[path] = new_idx
-
 
 def calculate_avg_char_width(font):
   sample_text = "abcdefghijklmnopqrstuvwxyz0123456789"
@@ -70,7 +64,7 @@ class MainController:
     self._create_global_themes()
     self.data_tree_view = DataTreeView(self.data_manager, self.ui_lock)
     self.plot_layout_manager = PlotLayoutManager(self.data_manager, self.playback_manager, scale=self.scale)
-    self.data_manager.add_callback(self.on_data_loaded)
+    self.data_manager.add_observer(self.on_data_loaded)
     self.avg_char_width = None
 
   def _create_global_themes(self):
@@ -86,11 +80,18 @@ class MainController:
         dpg.add_theme_color(dpg.mvPlotCol_Line, (255, 0, 0, 128), category=dpg.mvThemeCat_Plots)
 
   def on_data_loaded(self, data: dict):
-    self.playback_manager.set_route_duration(data['duration'])
-    num_msg_types = len(data['time_series_data'])
-    dpg.set_value("load_status", f"Loaded {num_msg_types} message types")
-    dpg.configure_item("load_button", enabled=True)
-    dpg.configure_item("timeline_slider", max_value=data['duration'])
+    duration = data.get('duration', 0.0)
+    self.playback_manager.set_route_duration(duration)
+
+    if data.get('loading_complete'):
+      num_paths = len(self.data_manager.get_all_paths())
+      dpg.set_value("load_status", f"Loaded {num_paths} data paths")
+      dpg.configure_item("load_button", enabled=True)
+    elif data.get('segment_added'):
+      segment_count = data.get('segment_count', 0)
+      dpg.set_value("load_status", f"Loading... {segment_count} segments processed")
+
+    dpg.configure_item("timeline_slider", max_value=duration)
 
   def setup_ui(self):
     with dpg.item_handler_registry(tag="tree_node_handler"):
@@ -179,11 +180,8 @@ class MainController:
       value_tag = f"value_{path}"
 
       if dpg.does_item_exist(value_tag) and dpg.is_item_visible(value_tag):
-        last_index = self.playback_manager.last_indices.get(path)
-        value, new_idx = self.data_manager.get_current_value(path, self.playback_manager.current_time_s, last_index)
-
+        value = self.data_manager.get_value_at(path, self.playback_manager.current_time_s)
         if value is not None:
-          self.playback_manager.update_index(path, new_idx)
           formatted_value = format_and_truncate(value, value_column_width, self.avg_char_width)
           dpg.set_value(value_tag, formatted_value)
 
diff --git a/tools/jotpluggler/views.py b/tools/jotpluggler/views.py
index feeff7ab24..feb8abac69 100644
--- a/tools/jotpluggler/views.py
+++ b/tools/jotpluggler/views.py
@@ -45,13 +45,13 @@ class TimeSeriesPanel(ViewPanel):
     self._ui_created = False
     self._preserved_series_data: list[tuple[str, tuple]] = []  # TODO: the way we do this right now doesn't make much sense
     self._series_legend_tags: dict[str, str] = {}  # Maps series_path to legend tag
-    self.data_manager.add_callback(self.on_data_loaded)
+    self.data_manager.add_observer(self.on_data_loaded)
 
   def preserve_data(self):
     self._preserved_series_data = []
     if self.plotted_series and self._ui_created:
       for series_path in self.plotted_series:
-        time_value_data = self.data_manager.get_time_series(series_path)
+        time_value_data = self.data_manager.get_timeseries(series_path)
         if time_value_data:
           self._preserved_series_data.append((series_path, time_value_data))
 
@@ -86,12 +86,9 @@ class TimeSeriesPanel(ViewPanel):
 
     if self.plotted_series:  # update legend labels with current values
       for series_path in self.plotted_series:
-        last_index = self.playback_manager.last_indices.get(series_path)
-        value, new_idx = self.data_manager.get_current_value(series_path, current_time_s, last_index)
+        value = self.data_manager.get_value_at(series_path, current_time_s)
 
         if value is not None:
-          self.playback_manager.update_index(series_path, new_idx)
-
           if isinstance(value, (int, float)):
             if isinstance(value, float):
               formatted_value = f"{value:.4f}" if abs(value) < 1000 else f"{value:.3e}"
@@ -100,7 +97,6 @@ class TimeSeriesPanel(ViewPanel):
           else:
             formatted_value = str(value)
 
-          # Update the series label to include current value
           series_tag = f"series_{self.panel_id}_{series_path.replace('/', '_')}"
           legend_label = f"{series_path}: {formatted_value}"
 
@@ -125,7 +121,6 @@ class TimeSeriesPanel(ViewPanel):
     if self.plot_tag and dpg.does_item_exist(self.plot_tag):
       dpg.delete_item(self.plot_tag)
 
-    # self.data_manager.remove_callback(self.on_data_loaded)
     self._series_legend_tags.clear()
     self._ui_created = False
 
@@ -136,7 +131,7 @@ class TimeSeriesPanel(ViewPanel):
     if series_path in self.plotted_series:
       return False
 
-    time_value_data = self.data_manager.get_time_series(series_path)
+    time_value_data = self.data_manager.get_timeseries(series_path)
     if time_value_data is None:
       return False
 
@@ -153,7 +148,7 @@ class TimeSeriesPanel(ViewPanel):
       if dpg.does_item_exist(series_tag):
         dpg.delete_item(series_tag)
       self.plotted_series.remove(series_path)
-      if series_path in self._series_legend_tags:  # Clean up legend tag mapping
+      if series_path in self._series_legend_tags:
         del self._series_legend_tags[series_path]
 
   def on_data_loaded(self, data: dict):
@@ -161,7 +156,7 @@ class TimeSeriesPanel(ViewPanel):
       self._update_series_data(series_path)
 
   def _update_series_data(self, series_path: str) -> bool:
-    time_value_data = self.data_manager.get_time_series(series_path)
+    time_value_data = self.data_manager.get_timeseries(series_path)
     if time_value_data is None:
       return False
 
@@ -196,7 +191,7 @@ class DataTreeView:
     self.current_search = ""
     self.data_tree = DataTreeNode(name="root")
     self.active_leaf_nodes: list[DataTreeNode] = []
-    self.data_manager.add_callback(self.on_data_loaded)
+    self.data_manager.add_observer(self.on_data_loaded)
 
   def on_data_loaded(self, data: dict):
     with self.ui_lock:
@@ -246,7 +241,7 @@ class DataTreeView:
 
     for child in sorted_children:
       if child.is_leaf:
-        is_plottable = self.data_manager.is_path_plottable(child.full_path)
+        is_plottable = self.data_manager.is_plottable(child.full_path)
 
         # Create draggable item
         with dpg.group(parent=parent_tag) as draggable_group:
@@ -266,10 +261,6 @@ class DataTreeView:
         node_tag = f"tree_{child.full_path}"
         label = child.name
 
-        if '/' not in child.full_path:
-          sample_count = len(self.data_manager.time_series_data.get(child.full_path, {}).get('t', []))
-          label = f"{child.name} ({sample_count} samples)"
-
         should_open = bool(search_term) and len(search_term) > 1 and any(search_term in path for path in self._get_all_descendant_paths(child))
 
         with dpg.tree_node(label=label, parent=parent_tag, tag=node_tag, default_open=should_open):

From 34f4281b9cd4f56936d3b8908509750e8fa30c60 Mon Sep 17 00:00:00 2001
From: "Quantizr (Jimmy)" <9859727+Quantizr@users.noreply.github.com>
Date: Wed, 27 Aug 2025 01:58:57 -0700
Subject: [PATCH 07/19] allow verrryyy long search results

---
 tools/jotpluggler/pluggle.py |  48 ++++++---
 tools/jotpluggler/views.py   | 201 ++++++++++++++++++++++-------------
 2 files changed, 164 insertions(+), 85 deletions(-)

diff --git a/tools/jotpluggler/pluggle.py b/tools/jotpluggler/pluggle.py
index 614bb53e8e..d7a2cfba34 100755
--- a/tools/jotpluggler/pluggle.py
+++ b/tools/jotpluggler/pluggle.py
@@ -66,6 +66,8 @@ class MainController:
     self.plot_layout_manager = PlotLayoutManager(self.data_manager, self.playback_manager, scale=self.scale)
     self.data_manager.add_observer(self.on_data_loaded)
     self.avg_char_width = None
+    self.visible_paths = set()
+    self.check_index = 0
 
   def _create_global_themes(self):
     with dpg.theme(tag="global_line_theme"):
@@ -94,9 +96,6 @@ class MainController:
     dpg.configure_item("timeline_slider", max_value=duration)
 
   def setup_ui(self):
-    with dpg.item_handler_registry(tag="tree_node_handler"):
-      dpg.add_item_toggled_open_handler(callback=self.data_tree_view.update_active_nodes_list)
-
     dpg.set_viewport_resize_callback(callback=self.on_viewport_resize)
 
     with dpg.window(tag="Primary Window"):
@@ -158,32 +157,53 @@ class MainController:
   def update_frame(self, font):
     with self.ui_lock:
       if self.avg_char_width is None:
-        self.avg_char_width = calculate_avg_char_width(font)  # must be calculated after first frame
+        self.avg_char_width = calculate_avg_char_width(font)
+      self.data_tree_view.update_frame()
 
       new_time = self.playback_manager.update_time(dpg.get_delta_time())
       if not dpg.is_item_active("timeline_slider"):
         dpg.set_value("timeline_slider", new_time)
 
       self._update_timeline_indicators(new_time)
+
       if not self.data_manager.loading and self.avg_char_width:
+        self._update_visible_set()
         self._update_data_values()
 
       dpg.set_value("fps_counter", f"{dpg.get_frame_rate():.1f} FPS")
 
+  def _update_visible_set(self): # for some reason, dpg has no way to easily check for visibility, and checking is slow...
+    all_paths = list(self.data_tree_view.created_leaf_paths)
+    if not all_paths:
+      self.visible_paths.clear()
+      return
+    chunk_size = min(50, len(all_paths)) # check up to 50 paths per frame
+    end_index = min(self.check_index + chunk_size, len(all_paths))
+    for i in range(self.check_index, end_index):
+      path = all_paths[i]
+      value_tag = f"value_{path}"
+      if dpg.does_item_exist(value_tag) and dpg.is_item_visible(value_tag):
+        self.visible_paths.add(path)
+      else:
+        self.visible_paths.discard(path)
+    self.check_index = end_index if end_index < len(all_paths) else 0
+
   def _update_data_values(self):
-    pool_width = dpg.get_item_rect_size("data_pool_window")[0]
-    value_column_width = pool_width * 0.5
-    active_nodes = self.data_tree_view.active_leaf_nodes
+    value_column_width = dpg.get_item_rect_size("data_pool_window")[0] // 2
 
-    for node in active_nodes:
-      path = node.full_path
+    for path in self.visible_paths.copy(): # avoid modification during iteration
       value_tag = f"value_{path}"
+      group_tag = f"group_{path}"
 
-      if dpg.does_item_exist(value_tag) and dpg.is_item_visible(value_tag):
-        value = self.data_manager.get_value_at(path, self.playback_manager.current_time_s)
-        if value is not None:
-          formatted_value = format_and_truncate(value, value_column_width, self.avg_char_width)
-          dpg.set_value(value_tag, formatted_value)
+      if not dpg.does_item_exist(value_tag) or not dpg.does_item_exist(group_tag):
+        self.visible_paths.discard(path)
+        continue
+
+      dpg.configure_item(group_tag, xoffset=value_column_width)
+      value = self.data_manager.get_value_at(path, self.playback_manager.current_time_s)
+      if value is not None:
+        formatted_value = format_and_truncate(value, value_column_width, self.avg_char_width)
+        dpg.set_value(value_tag, formatted_value)
 
   def _update_timeline_indicators(self, current_time_s: float):
     def update_node_recursive(node):
diff --git a/tools/jotpluggler/views.py b/tools/jotpluggler/views.py
index feb8abac69..bf089b4c6d 100644
--- a/tools/jotpluggler/views.py
+++ b/tools/jotpluggler/views.py
@@ -2,6 +2,7 @@ import os
 import re
 import uuid
 import threading
+from collections import deque
 import dearpygui.dearpygui as dpg
 from abc import ABC, abstractmethod
 from openpilot.tools.jotpluggler.data import DataManager
@@ -183,36 +184,68 @@ class DataTreeNode:
     self.full_path = full_path
     self.children: dict[str, DataTreeNode] = {}
     self.is_leaf = False
+    self.child_count = 0
+    self.is_plottable_cached = None
+    self.ui_created = False
+    self.ui_tag: str | None = None
+
 
 class DataTreeView:
+  MAX_ITEMS_PER_FRAME = 50
+
   def __init__(self, data_manager: DataManager, ui_lock: threading.Lock):
     self.data_manager = data_manager
     self.ui_lock = ui_lock
     self.current_search = ""
     self.data_tree = DataTreeNode(name="root")
-    self.active_leaf_nodes: list[DataTreeNode] = []
-    self.data_manager.add_observer(self.on_data_loaded)
-
-  def on_data_loaded(self, data: dict):
-    with self.ui_lock:
-      self.populate_data_tree()
+    self.ui_render_queue: deque[tuple[DataTreeNode, str, str, bool]] = deque() # (node, parent_tag, search_term, is_leaf)
+    self.visible_expanded_nodes: set[str] = set()
+    self.created_leaf_paths: set[str] = set()
+    self._all_paths_cache: list[str] = []
+    self.data_manager.add_observer(self._on_data_loaded)
+
+  def _on_data_loaded(self, data: dict):
+    if data.get('loading_complete'):
+      with self.ui_lock:
+        self._all_paths_cache = self.data_manager.get_all_paths()
+        self._populate_tree()
+
+  def update_frame(self):
+    items_processed = 0
+    while self.ui_render_queue and items_processed < self.MAX_ITEMS_PER_FRAME: # process up to MAX_ITEMS_PER_FRAME to maintain perforamnce
+      node, parent_tag, search_term, is_leaf = self.ui_render_queue.popleft()
+      if is_leaf:
+        self._create_leaf_ui(node, parent_tag)
+      else:
+        self._create_node_ui(node, parent_tag, search_term)
+      items_processed += 1
 
-  def populate_data_tree(self):
-    if not dpg.does_item_exist("data_tree_container"):
-      return
+  def search_data(self, search_term: str):
+    self.current_search = search_term
+    self._all_paths_cache = self.data_manager.get_all_paths()
+    self._populate_tree()
 
-    dpg.delete_item("data_tree_container", children_only=True)
+  def _populate_tree(self):
+    self._clear_ui()
     search_term = self.current_search.strip().lower()
+    self.data_tree = self._build_tree_structure(search_term)
+    for child in sorted(self.data_tree.children.values(), key=self._natural_sort_key): # queue top level nodes
+      self.ui_render_queue.append((child, "data_tree_container", search_term, child.is_leaf))
 
-    self.data_tree = DataTreeNode(name="root")
-    all_paths = self.data_manager.get_all_paths()
-
-    for path in sorted(all_paths):
-      if not self._should_display_path(path, search_term):
+  def _clear_ui(self):
+    dpg.delete_item("data_tree_container", children_only=True)
+    self.ui_render_queue.clear()
+    self.visible_expanded_nodes.clear()
+    self.created_leaf_paths.clear()
+
+  def _build_tree_structure(self, search_term: str) -> DataTreeNode:
+    root = DataTreeNode(name="root")
+    for path in sorted(self._all_paths_cache):
+      if not self._should_show_path(path, search_term):
         continue
 
       parts = path.split('/')
-      current_node = self.data_tree
+      current_node = root
       current_path_prefix = ""
 
       for part in parts:
@@ -220,76 +253,102 @@ class DataTreeView:
         if part not in current_node.children:
           current_node.children[part] = DataTreeNode(name=part, full_path=current_path_prefix)
         current_node = current_node.children[part]
-
       current_node.is_leaf = True
+    self._calculate_child_counts(root)
+    return root
 
-    self._create_ui_from_tree_recursive(self.data_tree, "data_tree_container", search_term)
-    self.update_active_nodes_list()
+  def _calculate_child_counts(self, node: DataTreeNode):
+    if node.is_leaf:
+      node.child_count = 0
+    else:
+      node.child_count = len(node.children)
+      for child in node.children.values():
+        self._calculate_child_counts(child)
 
-  def _should_display_path(self, path: str, search_term: str) -> bool:
-    if 'DEPRECATED' in path and not os.environ.get('SHOW_DEPRECATED'):
-      return False
-    return not search_term or search_term in path.lower()
+  def _create_node_ui(self, node: DataTreeNode, parent_tag: str, search_term: str):
+    if node.is_leaf:
+      self._create_leaf_ui(node, parent_tag)
+    else:
+      self._create_tree_node_ui(node, parent_tag, search_term)
 
-  def _natural_sort_key(self, node: DataTreeNode):
-    node_type_key = node.is_leaf
-    parts = [int(p) if p.isdigit() else p.lower() for p in re.split(r'(\d+)', node.name) if p]
-    return (node_type_key, parts)
+  def _create_tree_node_ui(self, node: DataTreeNode, parent_tag: str, search_term: str):
+    node_tag = f"tree_{node.full_path}"
+    node.ui_tag = node_tag
 
-  def _create_ui_from_tree_recursive(self, node: DataTreeNode, parent_tag: str, search_term: str):
-    sorted_children = sorted(node.children.values(), key=self._natural_sort_key)
+    label = f"{node.name} ({node.child_count} fields)"
+    should_open = (bool(search_term) and len(search_term) > 1 and any(search_term in path for path in self._get_descendant_paths(node)))
 
-    for child in sorted_children:
-      if child.is_leaf:
-        is_plottable = self.data_manager.is_plottable(child.full_path)
+    with dpg.tree_node(label=label, parent=parent_tag, tag=node_tag, default_open=should_open, open_on_arrow=True, open_on_double_click=True) as tree_node:
+      with dpg.item_handler_registry() as handler:
+        dpg.add_item_toggled_open_handler(callback=lambda s, d, u: self._on_node_expanded(node, search_term))
+      dpg.bind_item_handler_registry(tree_node, handler)
 
-        # Create draggable item
-        with dpg.group(parent=parent_tag) as draggable_group:
-          with dpg.table(header_row=False, borders_innerV=False, borders_outerH=False, borders_outerV=False, policy=dpg.mvTable_SizingStretchProp):
-            dpg.add_table_column(init_width_or_weight=0.5)
-            dpg.add_table_column(init_width_or_weight=0.5)
-            with dpg.table_row():
-              dpg.add_text(child.name)
-              dpg.add_text("N/A", tag=f"value_{child.full_path}")
+    node.ui_created = True
 
-        # Add drag payload if plottable
-        if is_plottable:
-          with dpg.drag_payload(parent=draggable_group, drag_data=child.full_path, payload_type="TIMESERIES_PAYLOAD"):
-            dpg.add_text(f"Plot: {child.full_path}")
+    if should_open:
+      self.visible_expanded_nodes.add(node.full_path)
+      self._queue_children(node, node_tag, search_term)
 
-      else:
-        node_tag = f"tree_{child.full_path}"
-        label = child.name
+  def _create_leaf_ui(self, node: DataTreeNode, parent_tag: str):
+    half_split_size = dpg.get_item_rect_size("data_pool_window")[0] // 2
+    with dpg.group(parent=parent_tag, horizontal=True, xoffset=half_split_size, tag=f"group_{node.full_path}") as draggable_group:
+      dpg.add_text(node.name)
+      dpg.add_text("N/A", tag=f"value_{node.full_path}")
+
+    if node.is_plottable_cached is None:
+      node.is_plottable_cached = self.data_manager.is_plottable(node.full_path)
+
+    if node.is_plottable_cached:
+      with dpg.drag_payload(parent=draggable_group, drag_data=node.full_path,
+                            payload_type="TIMESERIES_PAYLOAD"):
+        dpg.add_text(f"Plot: {node.full_path}")
 
-        should_open = bool(search_term) and len(search_term) > 1 and any(search_term in path for path in self._get_all_descendant_paths(child))
+    node.ui_created = True
+    node.ui_tag = f"value_{node.full_path}"
+    self.created_leaf_paths.add(node.full_path)
+
+  def _queue_children(self, node: DataTreeNode, parent_tag: str, search_term: str):
+    for child in sorted(node.children.values(), key=self._natural_sort_key):
+      self.ui_render_queue.append((child, parent_tag, search_term, child.is_leaf))
+
+  def _on_node_expanded(self, node: DataTreeNode, search_term: str):
+    node_tag = f"tree_{node.full_path}"
+    if not dpg.does_item_exist(node_tag):
+      return
+
+    is_expanded = dpg.get_value(node_tag)
+
+    if is_expanded:
+      if node.full_path not in self.visible_expanded_nodes:
+        self.visible_expanded_nodes.add(node.full_path)
+        self._queue_children(node, node_tag, search_term)
+    else:
+      self.visible_expanded_nodes.discard(node.full_path)
+      self._remove_children_from_queue(node.full_path)
+
+  def _remove_children_from_queue(self, collapsed_node_path: str):
+    new_queue = deque()
+    for node, parent_tag, search_term, is_leaf in self.ui_render_queue:
+      # Keep items that are not children of the collapsed node
+      if not node.full_path.startswith(collapsed_node_path + "/"):
+        new_queue.append((node, parent_tag, search_term, is_leaf))
+    self.ui_render_queue = new_queue
+
+  def _should_show_path(self, path: str, search_term: str) -> bool:
+    if 'DEPRECATED' in path and not os.environ.get('SHOW_DEPRECATED'):
+      return False
+    return not search_term or search_term in path.lower()
 
-        with dpg.tree_node(label=label, parent=parent_tag, tag=node_tag, default_open=should_open):
-          dpg.bind_item_handler_registry(node_tag, "tree_node_handler")
-          self._create_ui_from_tree_recursive(child, node_tag, search_term)
+  def _natural_sort_key(self, node: DataTreeNode):
+    node_type_key = node.is_leaf
+    parts = [int(p) if p.isdigit() else p.lower() for p in re.split(r'(\d+)', node.name) if p]
+    return (node_type_key, parts)
 
-  def _get_all_descendant_paths(self, node: DataTreeNode):
+  def _get_descendant_paths(self, node: DataTreeNode):
     for child_name, child_node in node.children.items():
       child_name_lower = child_name.lower()
       if child_node.is_leaf:
         yield child_name_lower
       else:
-        for path in self._get_all_descendant_paths(child_node):
+        for path in self._get_descendant_paths(child_node):
           yield f"{child_name_lower}/{path}"
-
-  def search_data(self, search_term: str):
-    self.current_search = search_term
-    self.populate_data_tree()
-
-  def update_active_nodes_list(self, sender=None, app_data=None, user_data=None):
-    self.active_leaf_nodes = self.get_active_leaf_nodes(self.data_tree)
-
-  def get_active_leaf_nodes(self, node: DataTreeNode):
-    active_leaves = []
-    for child in node.children.values():
-      if child.is_leaf:
-        active_leaves.append(child)
-      else:
-        node_tag = f"tree_{child.full_path}"
-        if dpg.does_item_exist(node_tag) and dpg.get_value(node_tag):
-          active_leaves.extend(self.get_active_leaf_nodes(child))
-    return active_leaves

From 789c470c643252a6925553d15d2311aa5187a5c3 Mon Sep 17 00:00:00 2001
From: "Quantizr (Jimmy)" <9859727+Quantizr@users.noreply.github.com>
Date: Wed, 27 Aug 2025 02:21:16 -0700
Subject: [PATCH 08/19] stream in multiprocessed segments

---
 tools/jotpluggler/data.py  |   2 +-
 tools/jotpluggler/views.py | 104 ++++++++++++++++++++++++++-----------
 2 files changed, 75 insertions(+), 31 deletions(-)

diff --git a/tools/jotpluggler/data.py b/tools/jotpluggler/data.py
index 57b6ac972c..1bdbc36b5a 100644
--- a/tools/jotpluggler/data.py
+++ b/tools/jotpluggler/data.py
@@ -289,7 +289,7 @@ class DataManager:
       observers = self._observers.copy()
 
     for callback in observers:
-      callback({'segment_added': True, 'duration': self._duration})
+      callback({'segment_added': True, 'duration': self._duration, 'segment_count': len(self._segments)})
 
   def _finalize_loading(self):
     with self._lock:
diff --git a/tools/jotpluggler/views.py b/tools/jotpluggler/views.py
index bf089b4c6d..9ae46b4ea0 100644
--- a/tools/jotpluggler/views.py
+++ b/tools/jotpluggler/views.py
@@ -202,13 +202,78 @@ class DataTreeView:
     self.visible_expanded_nodes: set[str] = set()
     self.created_leaf_paths: set[str] = set()
     self._all_paths_cache: list[str] = []
+    self._previous_paths_set: set[str] = set()
     self.data_manager.add_observer(self._on_data_loaded)
 
   def _on_data_loaded(self, data: dict):
-    if data.get('loading_complete'):
-      with self.ui_lock:
-        self._all_paths_cache = self.data_manager.get_all_paths()
-        self._populate_tree()
+    with self.ui_lock:
+      if data.get('segment_added'):
+        current_paths = set(self.data_manager.get_all_paths())
+        new_paths = current_paths - self._previous_paths_set
+        if new_paths:
+          self._all_paths_cache = list(current_paths)
+          if not self._previous_paths_set:
+            self._populate_tree()
+          else:
+            self._add_paths_to_tree(new_paths, incremental=True)
+          self._previous_paths_set = current_paths.copy()
+
+  def _populate_tree(self):
+    self._clear_ui()
+    search_term = self.current_search.strip().lower()
+    self.data_tree = self._add_paths_to_tree(self._all_paths_cache, incremental=False)
+    for child in sorted(self.data_tree.children.values(), key=self._natural_sort_key):
+      self.ui_render_queue.append((child, "data_tree_container", search_term, child.is_leaf))
+
+
+  def _add_paths_to_tree(self, paths, incremental=False):
+    search_term = self.current_search.strip().lower()
+    filtered_paths = [path for path in paths if self._should_show_path(path, search_term)]
+
+    if not filtered_paths:
+      return
+
+    nodes_to_update = set() if incremental else None
+    target_tree = self.data_tree if incremental else DataTreeNode(name="root")
+
+    for path in sorted(filtered_paths):
+      parts = path.split('/')
+      current_node = target_tree
+      current_path_prefix = ""
+
+      for i, part in enumerate(parts):
+        current_path_prefix = f"{current_path_prefix}/{part}" if current_path_prefix else part
+
+        if part not in current_node.children:
+          current_node.children[part] = DataTreeNode(name=part, full_path=current_path_prefix)
+          if incremental:
+            nodes_to_update.add(current_node)
+
+        current_node = current_node.children[part]
+        if incremental and i < len(parts) - 1:
+          nodes_to_update.add(current_node)
+
+      if not current_node.is_leaf:
+        current_node.is_leaf = True
+        if incremental:
+          nodes_to_update.add(current_node)
+
+    self._calculate_child_counts(target_tree)
+    if incremental:
+      self._queue_new_ui_items(filtered_paths, search_term)
+    return target_tree
+
+  def _queue_new_ui_items(self, new_paths, search_term):
+    for path in new_paths:
+      parts = path.split('/')
+      parent_path = '/'.join(parts[:-1]) if len(parts) > 1 else ""
+      if parent_path == "" or parent_path in self.visible_expanded_nodes:
+        parent_tag = "data_tree_container" if parent_path == "" else f"tree_{parent_path}"
+        if dpg.does_item_exist(parent_tag):
+          node = self.data_tree
+          for part in parts:
+            node = node.children[part]
+          self.ui_render_queue.append((node, parent_tag, search_term, True))
 
   def update_frame(self):
     items_processed = 0
@@ -223,40 +288,15 @@ class DataTreeView:
   def search_data(self, search_term: str):
     self.current_search = search_term
     self._all_paths_cache = self.data_manager.get_all_paths()
+    self._previous_paths_set = set(self._all_paths_cache)  # Reset tracking after search
     self._populate_tree()
 
-  def _populate_tree(self):
-    self._clear_ui()
-    search_term = self.current_search.strip().lower()
-    self.data_tree = self._build_tree_structure(search_term)
-    for child in sorted(self.data_tree.children.values(), key=self._natural_sort_key): # queue top level nodes
-      self.ui_render_queue.append((child, "data_tree_container", search_term, child.is_leaf))
-
   def _clear_ui(self):
     dpg.delete_item("data_tree_container", children_only=True)
     self.ui_render_queue.clear()
     self.visible_expanded_nodes.clear()
     self.created_leaf_paths.clear()
 
-  def _build_tree_structure(self, search_term: str) -> DataTreeNode:
-    root = DataTreeNode(name="root")
-    for path in sorted(self._all_paths_cache):
-      if not self._should_show_path(path, search_term):
-        continue
-
-      parts = path.split('/')
-      current_node = root
-      current_path_prefix = ""
-
-      for part in parts:
-        current_path_prefix = f"{current_path_prefix}/{part}" if current_path_prefix else part
-        if part not in current_node.children:
-          current_node.children[part] = DataTreeNode(name=part, full_path=current_path_prefix)
-        current_node = current_node.children[part]
-      current_node.is_leaf = True
-    self._calculate_child_counts(root)
-    return root
-
   def _calculate_child_counts(self, node: DataTreeNode):
     if node.is_leaf:
       node.child_count = 0
@@ -272,6 +312,8 @@ class DataTreeView:
       self._create_tree_node_ui(node, parent_tag, search_term)
 
   def _create_tree_node_ui(self, node: DataTreeNode, parent_tag: str, search_term: str):
+    if not dpg.does_item_exist(parent_tag):
+      return
     node_tag = f"tree_{node.full_path}"
     node.ui_tag = node_tag
 
@@ -290,6 +332,8 @@ class DataTreeView:
       self._queue_children(node, node_tag, search_term)
 
   def _create_leaf_ui(self, node: DataTreeNode, parent_tag: str):
+    if not dpg.does_item_exist(parent_tag):
+      return
     half_split_size = dpg.get_item_rect_size("data_pool_window")[0] // 2
     with dpg.group(parent=parent_tag, horizontal=True, xoffset=half_split_size, tag=f"group_{node.full_path}") as draggable_group:
       dpg.add_text(node.name)

From 3447acde4152b6381d42b81fe204c45278cb203e Mon Sep 17 00:00:00 2001
From: "Quantizr (Jimmy)" <9859727+Quantizr@users.noreply.github.com>
Date: Wed, 27 Aug 2025 17:09:34 -0700
Subject: [PATCH 09/19] bug fixes

---
 tools/jotpluggler/data.py    |  5 ++---
 tools/jotpluggler/pluggle.py |  2 +-
 tools/jotpluggler/views.py   | 19 +++++++++----------
 3 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/tools/jotpluggler/data.py b/tools/jotpluggler/data.py
index 1bdbc36b5a..e8572de0e9 100644
--- a/tools/jotpluggler/data.py
+++ b/tools/jotpluggler/data.py
@@ -3,7 +3,6 @@ import threading
 import multiprocessing
 import bisect
 from collections import defaultdict
-from typing import Any
 import tqdm
 from openpilot.common.swaglog import cloudlog
 from openpilot.tools.lib.logreader import _LogFileReader, LogReader
@@ -11,7 +10,7 @@ from openpilot.tools.lib.logreader import _LogFileReader, LogReader
 
 def flatten_dict(d: dict, sep: str = "/", prefix: str = None) -> dict:
   result = {}
-  stack = [(d, prefix)]
+  stack: list[tuple] = [(d, prefix)]
 
   while stack:
     obj, current_prefix = stack.pop()
@@ -164,7 +163,7 @@ def msgs_to_time_series(msgs):
   return final_result, min_time or 0.0, max_time or 0.0
 
 
-def _process_segment(segment_identifier: str) -> tuple[dict[str, Any], float, float]:
+def _process_segment(segment_identifier: str):
   try:
     lr = _LogFileReader(segment_identifier, sort_by_time=True)
     return msgs_to_time_series(lr)
diff --git a/tools/jotpluggler/pluggle.py b/tools/jotpluggler/pluggle.py
index d7a2cfba34..f55683633b 100755
--- a/tools/jotpluggler/pluggle.py
+++ b/tools/jotpluggler/pluggle.py
@@ -66,7 +66,7 @@ class MainController:
     self.plot_layout_manager = PlotLayoutManager(self.data_manager, self.playback_manager, scale=self.scale)
     self.data_manager.add_observer(self.on_data_loaded)
     self.avg_char_width = None
-    self.visible_paths = set()
+    self.visible_paths: set[str] = set()
     self.check_index = 0
 
   def _create_global_themes(self):
diff --git a/tools/jotpluggler/views.py b/tools/jotpluggler/views.py
index 9ae46b4ea0..4577a8f7ed 100644
--- a/tools/jotpluggler/views.py
+++ b/tools/jotpluggler/views.py
@@ -185,7 +185,7 @@ class DataTreeNode:
     self.children: dict[str, DataTreeNode] = {}
     self.is_leaf = False
     self.child_count = 0
-    self.is_plottable_cached = None
+    self.is_plottable_cached: bool | None = None
     self.ui_created = False
     self.ui_tag: str | None = None
 
@@ -229,12 +229,12 @@ class DataTreeView:
   def _add_paths_to_tree(self, paths, incremental=False):
     search_term = self.current_search.strip().lower()
     filtered_paths = [path for path in paths if self._should_show_path(path, search_term)]
+    target_tree = self.data_tree if incremental else DataTreeNode(name="root")
 
     if not filtered_paths:
-      return
+      return target_tree
 
     nodes_to_update = set() if incremental else None
-    target_tree = self.data_tree if incremental else DataTreeNode(name="root")
 
     for path in sorted(filtered_paths):
       parts = path.split('/')
@@ -339,13 +339,12 @@ class DataTreeView:
       dpg.add_text(node.name)
       dpg.add_text("N/A", tag=f"value_{node.full_path}")
 
-    if node.is_plottable_cached is None:
-      node.is_plottable_cached = self.data_manager.is_plottable(node.full_path)
+      if node.is_plottable_cached is None:
+        node.is_plottable_cached = self.data_manager.is_plottable(node.full_path)
 
-    if node.is_plottable_cached:
-      with dpg.drag_payload(parent=draggable_group, drag_data=node.full_path,
-                            payload_type="TIMESERIES_PAYLOAD"):
-        dpg.add_text(f"Plot: {node.full_path}")
+      if node.is_plottable_cached:
+        with dpg.drag_payload(parent=draggable_group, drag_data=node.full_path, payload_type="TIMESERIES_PAYLOAD"):
+          dpg.add_text(f"Plot: {node.full_path}")
 
     node.ui_created = True
     node.ui_tag = f"value_{node.full_path}"
@@ -371,7 +370,7 @@ class DataTreeView:
       self._remove_children_from_queue(node.full_path)
 
   def _remove_children_from_queue(self, collapsed_node_path: str):
-    new_queue = deque()
+    new_queue: deque[tuple] = deque()
     for node, parent_tag, search_term, is_leaf in self.ui_render_queue:
       # Keep items that are not children of the collapsed node
       if not node.full_path.startswith(collapsed_node_path + "/"):

From 9a9a6586d48c4bc3810ffb605e9cd67cb47da9f8 Mon Sep 17 00:00:00 2001
From: "Quantizr (Jimmy)" <9859727+Quantizr@users.noreply.github.com>
Date: Wed, 27 Aug 2025 20:43:07 -0700
Subject: [PATCH 10/19] simplify and speed up timeseries

---
 tools/jotpluggler/data.py    |   2 +-
 tools/jotpluggler/layout.py  |  18 -----
 tools/jotpluggler/pluggle.py |   2 +-
 tools/jotpluggler/views.py   | 129 +++++++++++------------------------
 4 files changed, 42 insertions(+), 109 deletions(-)

diff --git a/tools/jotpluggler/data.py b/tools/jotpluggler/data.py
index e8572de0e9..b4ceb9affe 100644
--- a/tools/jotpluggler/data.py
+++ b/tools/jotpluggler/data.py
@@ -200,7 +200,7 @@ class DataManager:
           values.append(segment[msg_type][field])
 
       if not times:
-        return None
+        return [], []
 
       combined_times = np.concatenate(times) - self._start_time
       if len(values) > 1 and any(arr.dtype != values[0].dtype for arr in values):
diff --git a/tools/jotpluggler/layout.py b/tools/jotpluggler/layout.py
index 0961635810..b113d4076d 100644
--- a/tools/jotpluggler/layout.py
+++ b/tools/jotpluggler/layout.py
@@ -18,10 +18,6 @@ class LayoutNode(ABC):
   def destroy_ui(self):
     pass
 
-  @abstractmethod
-  def preserve_data(self):
-    pass
-
 
 class LeafNode(LayoutNode):
   """Leaf node that contains a single ViewPanel with controls"""
@@ -32,9 +28,6 @@ class LeafNode(LayoutNode):
     self.layout_manager = layout_manager
     self.scale = scale
 
-  def preserve_data(self):
-    self.panel.preserve_data()
-
   def create_ui(self, parent_tag: str, width: int = -1, height: int = -1):
     """Create UI container with controls and panel"""
     self.tag = f"leaf_{self.node_id}"
@@ -95,10 +88,6 @@ class SplitterNode(LayoutNode):
     self.child_proportions = [1.0 / len(self.children) for _ in self.children] if self.children else []
     self.child_container_tags: list[str] = []  # Track container tags for resizing
 
-  def preserve_data(self):
-    for child in self.children:
-      child.preserve_data()
-
   def add_child(self, child: LayoutNode, index: int = None):
     if index is None:
       self.children.append(child)
@@ -219,14 +208,12 @@ class PlotLayoutManager:
     parent_node, child_index = self._find_parent_and_index(node)
 
     if parent_node is None:  # root node - create new splitter as root
-      node.preserve_data()
       self.root_node = SplitterNode([node, new_leaf], orientation)
       self._update_ui_for_node(self.root_node, self.container_tag)
     elif isinstance(parent_node, SplitterNode) and parent_node.orientation == orientation:  # same orientation - add to existing splitter
       parent_node.add_child(new_leaf, child_index + 1)
       self._update_ui_for_node(parent_node)
     else:  # different orientation - replace node with new splitter
-      node.preserve_data()
       new_splitter = SplitterNode([node, new_leaf], orientation)
       self._replace_child_in_parent(parent_node, node, new_splitter)
 
@@ -244,7 +231,6 @@ class PlotLayoutManager:
         grandparent_node, parent_index = self._find_parent_and_index(parent_node)
 
         if grandparent_node is None:  # promote remaining child to root
-          remaining_child.preserve_data()
           parent_node.children.remove(remaining_child)
           self.root_node = remaining_child
           parent_node.destroy_ui()
@@ -255,8 +241,6 @@ class PlotLayoutManager:
         self._update_ui_for_node(parent_node)
 
   def _replace_child_in_parent(self, parent_node: SplitterNode, old_child: LayoutNode, new_child: LayoutNode):
-    old_child.preserve_data()  # save data and for when recreating ui for the node
-
     child_index = parent_node.children.index(old_child)
     child_container_tag = f"child_container_{parent_node.node_id}_{child_index}"
 
@@ -272,8 +256,6 @@ class PlotLayoutManager:
       new_child.create_ui(child_container_tag, container_width, container_height)
 
   def _update_ui_for_node(self, node: LayoutNode, container_tag: str = None):
-    node.preserve_data()
-
     if container_tag:  #  update node in a specific container (usually root)
       dpg.delete_item(container_tag, children_only=True)
       container_width, container_height = dpg.get_item_rect_size(container_tag)
diff --git a/tools/jotpluggler/pluggle.py b/tools/jotpluggler/pluggle.py
index f55683633b..b370ea69bc 100755
--- a/tools/jotpluggler/pluggle.py
+++ b/tools/jotpluggler/pluggle.py
@@ -166,7 +166,7 @@ class MainController:
 
       self._update_timeline_indicators(new_time)
 
-      if not self.data_manager.loading and self.avg_char_width:
+      if self.avg_char_width:
         self._update_visible_set()
         self._update_data_values()
 
diff --git a/tools/jotpluggler/views.py b/tools/jotpluggler/views.py
index 4577a8f7ed..87889417f6 100644
--- a/tools/jotpluggler/views.py
+++ b/tools/jotpluggler/views.py
@@ -2,6 +2,7 @@ import os
 import re
 import uuid
 import threading
+import numpy as np
 from collections import deque
 import dearpygui.dearpygui as dpg
 from abc import ABC, abstractmethod
@@ -27,10 +28,6 @@ class ViewPanel(ABC):
   def get_panel_type(self) -> str:
     pass
 
-  @abstractmethod
-  def preserve_data(self):
-    pass
-
 
 class TimeSeriesPanel(ViewPanel):
   def __init__(self, data_manager: DataManager, playback_manager, panel_id: str | None = None):
@@ -38,24 +35,14 @@ class TimeSeriesPanel(ViewPanel):
     self.data_manager = data_manager
     self.playback_manager = playback_manager
     self.title = "Time Series Plot"
-    self.plotted_series: set[str] = set()
     self.plot_tag: str | None = None
     self.x_axis_tag: str | None = None
     self.y_axis_tag: str | None = None
     self.timeline_indicator_tag: str | None = None
     self._ui_created = False
-    self._preserved_series_data: list[tuple[str, tuple]] = []  # TODO: the way we do this right now doesn't make much sense
-    self._series_legend_tags: dict[str, str] = {}  # Maps series_path to legend tag
+    self._series_data: dict[str, tuple] = {}
     self.data_manager.add_observer(self.on_data_loaded)
 
-  def preserve_data(self):
-    self._preserved_series_data = []
-    if self.plotted_series and self._ui_created:
-      for series_path in self.plotted_series:
-        time_value_data = self.data_manager.get_timeseries(series_path)
-        if time_value_data:
-          self._preserved_series_data.append((series_path, time_value_data))
-
   def create_ui(self, parent_tag: str):
     self.plot_tag = f"plot_{self.panel_id}"
     self.x_axis_tag = f"{self.plot_tag}_x_axis"
@@ -64,18 +51,13 @@ class TimeSeriesPanel(ViewPanel):
 
     with dpg.plot(height=-1, width=-1, tag=self.plot_tag, parent=parent_tag, drop_callback=self._on_series_drop, payload_type="TIMESERIES_PAYLOAD"):
       dpg.add_plot_legend()
-      dpg.add_plot_axis(dpg.mvXAxis, label="", tag=self.x_axis_tag)
-      dpg.add_plot_axis(dpg.mvYAxis, label="", tag=self.y_axis_tag)
-
+      dpg.add_plot_axis(dpg.mvXAxis, no_label=True, tag=self.x_axis_tag)
+      dpg.add_plot_axis(dpg.mvYAxis, no_label=True, tag=self.y_axis_tag)
       timeline_series_tag = dpg.add_inf_line_series(x=[0], label="Timeline", parent=self.y_axis_tag, tag=self.timeline_indicator_tag)
       dpg.bind_item_theme(timeline_series_tag, "global_timeline_theme")
 
-    # Restore series from preserved data
-    if self._preserved_series_data:
-      self.plotted_series.clear()
-      for series_path, (rel_time_array, value_array) in self._preserved_series_data:
-        self._add_series_with_data(series_path, rel_time_array, value_array)
-      self._preserved_series_data = []
+    for series_path in list(self._series_data.keys()):
+      self.add_series(series_path)
 
     self._ui_created = True
 
@@ -83,99 +65,68 @@ class TimeSeriesPanel(ViewPanel):
     if not self._ui_created or not dpg.does_item_exist(self.timeline_indicator_tag):
       return
 
-    dpg.set_value(self.timeline_indicator_tag, [[current_time_s], [0]])  # vertical line position
+    dpg.set_value(self.timeline_indicator_tag, [[current_time_s], [0]])
 
-    if self.plotted_series:  # update legend labels with current values
-      for series_path in self.plotted_series:
-        value = self.data_manager.get_value_at(series_path, current_time_s)
+    for series_path, (rel_time_array, value_array) in self._series_data.items():
+      position = np.searchsorted(rel_time_array, current_time_s, side='right') - 1
+      value = None
 
-        if value is not None:
-          if isinstance(value, (int, float)):
-            if isinstance(value, float):
-              formatted_value = f"{value:.4f}" if abs(value) < 1000 else f"{value:.3e}"
-            else:
-              formatted_value = str(value)
-          else:
-            formatted_value = str(value)
+      if position >= 0 and (current_time_s - rel_time_array[position]) <= 1.0:
+        value = value_array[position]
 
-          series_tag = f"series_{self.panel_id}_{series_path.replace('/', '_')}"
-          legend_label = f"{series_path}: {formatted_value}"
+      if value is not None:
+        if isinstance(value, float):
+          formatted_value = f"{value:.4f}" if abs(value) < 1000 else f"{value:.3e}"
+        else:
+          formatted_value = str(value)
 
-          if dpg.does_item_exist(series_tag):
-            dpg.configure_item(series_tag, label=legend_label)
+        series_tag = f"series_{self.panel_id}_{series_path}"
+        legend_label = f"{series_path}: {formatted_value}"
 
-  def _add_series_with_data(self, series_path: str, rel_time_array, value_array) -> bool:
-    if series_path in self.plotted_series:
-      return False
+        if dpg.does_item_exist(series_tag):
+          dpg.configure_item(series_tag, label=legend_label)
 
-    series_tag = f"series_{self.panel_id}_{series_path.replace('/', '_')}"
-    line_series_tag = dpg.add_line_series(x=rel_time_array.tolist(), y=value_array.tolist(), label=series_path, parent=self.y_axis_tag, tag=series_tag)
+  def add_series(self, series_path: str, update: bool = False) -> bool:
+    if update or series_path not in self._series_data:
+      self._series_data[series_path] = self.data_manager.get_timeseries(series_path)
 
-    dpg.bind_item_theme(line_series_tag, "global_line_theme")
+    rel_time_array, value_array = self._series_data[series_path]
+    series_tag = f"series_{self.panel_id}_{series_path}"
 
-    self.plotted_series.add(series_path)
-    dpg.fit_axis_data(self.x_axis_tag)
-    dpg.fit_axis_data(self.y_axis_tag)
+    if dpg.does_item_exist(series_tag):
+      dpg.set_value(series_tag, [rel_time_array, value_array])
+    else:
+      line_series_tag = dpg.add_line_series(x=rel_time_array, y=value_array, label=series_path, parent=self.y_axis_tag, tag=series_tag)
+      dpg.bind_item_theme(line_series_tag, "global_line_theme")
+      dpg.fit_axis_data(self.x_axis_tag)
+      dpg.fit_axis_data(self.y_axis_tag)
     return True
 
   def destroy_ui(self):
     if self.plot_tag and dpg.does_item_exist(self.plot_tag):
       dpg.delete_item(self.plot_tag)
-
-    self._series_legend_tags.clear()
     self._ui_created = False
 
   def get_panel_type(self) -> str:
     return "timeseries"
 
-  def add_series(self, series_path: str) -> bool:
-    if series_path in self.plotted_series:
-      return False
-
-    time_value_data = self.data_manager.get_timeseries(series_path)
-    if time_value_data is None:
-      return False
-
-    rel_time_array, value_array = time_value_data
-    return self._add_series_with_data(series_path, rel_time_array, value_array)
-
   def clear_all_series(self):
-    for series_path in self.plotted_series.copy():
+    for series_path in list(self._series_data.keys()):
       self.remove_series(series_path)
 
   def remove_series(self, series_path: str):
-    if series_path in self.plotted_series:
-      series_tag = f"series_{self.panel_id}_{series_path.replace('/', '_')}"
+    if series_path in self._series_data:
+      series_tag = f"series_{self.panel_id}_{series_path}"
       if dpg.does_item_exist(series_tag):
         dpg.delete_item(series_tag)
-      self.plotted_series.remove(series_path)
-      if series_path in self._series_legend_tags:
-        del self._series_legend_tags[series_path]
+      del self._series_data[series_path]
 
   def on_data_loaded(self, data: dict):
-    for series_path in self.plotted_series.copy():
-      self._update_series_data(series_path)
-
-  def _update_series_data(self, series_path: str) -> bool:
-    time_value_data = self.data_manager.get_timeseries(series_path)
-    if time_value_data is None:
-      return False
-
-    rel_time_array, value_array = time_value_data
-    series_tag = f"series_{self.panel_id}_{series_path.replace('/', '_')}"
-
-    if dpg.does_item_exist(series_tag):
-      dpg.set_value(series_tag, [rel_time_array.tolist(), value_array.tolist()])
-      dpg.fit_axis_data(self.x_axis_tag)
-      dpg.fit_axis_data(self.y_axis_tag)
-      return True
-    else:
-      self.plotted_series.discard(series_path)
-      return False
+    for series_path in list(self._series_data.keys()):
+      self.add_series(series_path, update=True)
 
   def _on_series_drop(self, sender, app_data, user_data):
-    series_path = app_data
-    self.add_series(series_path)
+    self.add_series(app_data)
 
 
 class DataTreeNode:

From 1cfa906201ce88a876c4aba02e7f9d94293419c7 Mon Sep 17 00:00:00 2001
From: "Quantizr (Jimmy)" <9859727+Quantizr@users.noreply.github.com>
Date: Thu, 28 Aug 2025 00:06:30 -0700
Subject: [PATCH 11/19] small fixes

---
 tools/jotpluggler/data.py    | 2 +-
 tools/jotpluggler/pluggle.py | 3 ++-
 tools/jotpluggler/views.py   | 6 +++---
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/tools/jotpluggler/data.py b/tools/jotpluggler/data.py
index b4ceb9affe..0cbc62bf13 100644
--- a/tools/jotpluggler/data.py
+++ b/tools/jotpluggler/data.py
@@ -266,7 +266,7 @@ class DataManager:
           pbar.update(1)
           if segment_result:
             self._add_segment(segment_result, start_time, end_time)
-    except Exception as e:
+    except Exception:
       cloudlog.exception(f"Error loading route {route}:")
     finally:
       self._finalize_loading()
diff --git a/tools/jotpluggler/pluggle.py b/tools/jotpluggler/pluggle.py
index b370ea69bc..a44fa2ed3c 100755
--- a/tools/jotpluggler/pluggle.py
+++ b/tools/jotpluggler/pluggle.py
@@ -5,6 +5,7 @@ import pyautogui
 import subprocess
 import dearpygui.dearpygui as dpg
 import threading
+import numpy as np
 from openpilot.common.basedir import BASEDIR
 from openpilot.tools.jotpluggler.data import DataManager
 from openpilot.tools.jotpluggler.views import DataTreeView
@@ -48,7 +49,7 @@ def calculate_avg_char_width(font):
 
 
 def format_and_truncate(value, available_width: float, avg_char_width: float) -> str:
-  s = str(value)
+  s = f"{value:.5f}" if np.issubdtype(type(value), np.floating) else str(value)
   max_chars = int(available_width / avg_char_width) - 3
   if len(s) > max_chars:
     return s[: max(0, max_chars)] + "..."
diff --git a/tools/jotpluggler/views.py b/tools/jotpluggler/views.py
index 87889417f6..fd9fd09007 100644
--- a/tools/jotpluggler/views.py
+++ b/tools/jotpluggler/views.py
@@ -75,8 +75,8 @@ class TimeSeriesPanel(ViewPanel):
         value = value_array[position]
 
       if value is not None:
-        if isinstance(value, float):
-          formatted_value = f"{value:.4f}" if abs(value) < 1000 else f"{value:.3e}"
+        if np.issubdtype(type(value), np.floating):
+          formatted_value = f"{value:.5f}"
         else:
           formatted_value = str(value)
 
@@ -228,7 +228,7 @@ class DataTreeView:
 
   def update_frame(self):
     items_processed = 0
-    while self.ui_render_queue and items_processed < self.MAX_ITEMS_PER_FRAME: # process up to MAX_ITEMS_PER_FRAME to maintain perforamnce
+    while self.ui_render_queue and items_processed < self.MAX_ITEMS_PER_FRAME: # process up to MAX_ITEMS_PER_FRAME to maintain performance
       node, parent_tag, search_term, is_leaf = self.ui_render_queue.popleft()
       if is_leaf:
         self._create_leaf_ui(node, parent_tag)

From 628e19ffb07808c18aee461a725624d79992d8e8 Mon Sep 17 00:00:00 2001
From: "Quantizr (Jimmy)" <9859727+Quantizr@users.noreply.github.com>
Date: Fri, 29 Aug 2025 00:40:33 -0700
Subject: [PATCH 12/19] rewrite layout

---
 tools/jotpluggler/layout.py  | 430 ++++++++++++++---------------------
 tools/jotpluggler/pluggle.py |  19 +-
 tools/jotpluggler/views.py   |  14 +-
 3 files changed, 188 insertions(+), 275 deletions(-)

diff --git a/tools/jotpluggler/layout.py b/tools/jotpluggler/layout.py
index b113d4076d..683bf70a22 100644
--- a/tools/jotpluggler/layout.py
+++ b/tools/jotpluggler/layout.py
@@ -1,282 +1,198 @@
-import uuid
 import dearpygui.dearpygui as dpg
-from abc import ABC, abstractmethod
 from openpilot.tools.jotpluggler.data import DataManager
-from openpilot.tools.jotpluggler.views import ViewPanel, TimeSeriesPanel
+from openpilot.tools.jotpluggler.views import TimeSeriesPanel
 
 
-class LayoutNode(ABC):
-  def __init__(self, node_id: str | None = None):
-    self.node_id = node_id or str(uuid.uuid4())
-    self.tag: str | None = None
-
-  @abstractmethod
-  def create_ui(self, parent_tag: str, width: int = -1, height: int = -1):
-    pass
-
-  @abstractmethod
-  def destroy_ui(self):
-    pass
-
-
-class LeafNode(LayoutNode):
-  """Leaf node that contains a single ViewPanel with controls"""
-
-  def __init__(self, panel: ViewPanel, layout_manager=None, scale: float = 1.0, node_id: str = None):
-    super().__init__(node_id)
-    self.panel = panel
-    self.layout_manager = layout_manager
+class PlotLayoutManager:
+  def __init__(self, data_manager: DataManager, playback_manager, scale: float = 1.0):
+    self.data_manager = data_manager
+    self.playback_manager = playback_manager
     self.scale = scale
+    self.container_tag = "plot_layout_container"
+    self.active_panels = []
 
-  def create_ui(self, parent_tag: str, width: int = -1, height: int = -1):
-    """Create UI container with controls and panel"""
-    self.tag = f"leaf_{self.node_id}"
-
-    with dpg.child_window(tag=self.tag, parent=parent_tag, border=True, width=-1, height=-1, no_scrollbar=True):
-      # Control bar
-      with dpg.group(horizontal=True):
-        dpg.add_input_text(tag=f"title_{self.node_id}", default_value=self.panel.title, width=int(100 * self.scale), callback=self._on_title_change)
-        dpg.add_combo(
-          items=["Time Series"],  # "Camera", "Text Log", "Map View"],
-          tag=f"type_{self.node_id}",
-          default_value="Time Series",
-          width=int(100 * self.scale),
-          callback=self._on_type_change,
-        )
-        dpg.add_button(label="Clear", callback=self._clear, width=int(50 * self.scale))
-        dpg.add_button(label="Delete", callback=self._delete, width=int(50 * self.scale))
-        dpg.add_button(label="Split H", callback=lambda: self._split("horizontal"), width=int(50 * self.scale))
-        dpg.add_button(label="Split V", callback=lambda: self._split("vertical"), width=int(50 * self.scale))
-
-      dpg.add_separator()
-
-      # Panel content area
-      panel_area_tag = f"panel_area_{self.node_id}"
-      with dpg.child_window(tag=panel_area_tag, border=False, height=-1, width=-1, no_scrollbar=True):
-        self.panel.create_ui(panel_area_tag)
-
-  def destroy_ui(self):
-    if self.panel:
-      self.panel.destroy_ui()
-    if self.tag and dpg.does_item_exist(self.tag):
-      dpg.delete_item(self.tag)
-
-  def _on_title_change(self, sender, app_data):
-    self.panel.title = app_data
-
-  def _on_type_change(self, sender, app_data):
-    print(f"Panel type change requested: {app_data}")
+    initial_panel = TimeSeriesPanel(data_manager, playback_manager)
+    self.active_panels.append(initial_panel)
+    self.layout = {"type": "panel", "panel": initial_panel}
 
-  def _split(self, orientation: str):
-    if self.layout_manager:
-      self.layout_manager.split_node(self, orientation)
+  def create_ui(self, parent_tag: str):
+    if dpg.does_item_exist(self.container_tag):
+      dpg.delete_item(self.container_tag)
 
-  def _clear(self):
-    if hasattr(self.panel, 'clear_all_series'):
-      self.panel.clear_all_series()
+    with dpg.child_window(tag=self.container_tag, parent=parent_tag, border=False, width=-1, height=-1, no_scrollbar=True):
+      container_width, container_height = dpg.get_item_rect_size(self.container_tag)
+      self._create_ui_recursive(self.layout, self.container_tag, [], container_width, container_height)
 
-  def _delete(self):
-    if self.layout_manager:
-      self.layout_manager.delete_node(self)
+  def on_viewport_resize(self):
+    self._resize_splits_recursive(self.layout, [])
+
+  def split_panel(self, panel_path: list[int], orientation: str):
+    current_layout = self._get_layout_at_path(panel_path)
+    existing_panel = current_layout["panel"]
+    new_panel = TimeSeriesPanel(self.data_manager, self.playback_manager)
+    self.active_panels.append(new_panel)
+
+    parent, child_index = self._get_parent_and_index(panel_path)
+
+    if parent is None:  # Root split
+      self.layout = {
+        "type": "split",
+        "orientation": orientation,
+        "children": [{"type": "panel", "panel": existing_panel}, {"type": "panel", "panel": new_panel}],
+        "proportions": [0.5, 0.5],
+      }
+      self._rebuild_ui_at_path([])
+    elif parent["type"] == "split" and parent["orientation"] == orientation:
+      parent["children"].insert(child_index + 1, {"type": "panel", "panel": new_panel})
+      parent["proportions"] = [1.0 / len(parent["children"])] * len(parent["children"])
+      self._rebuild_ui_at_path(panel_path[:-1])
+    else:
+      new_split = {"type": "split", "orientation": orientation, "children": [current_layout, {"type": "panel", "panel": new_panel}], "proportions": [0.5, 0.5]}
+      self._replace_layout_at_path(panel_path, new_split)
+      self._rebuild_ui_at_path(panel_path)
+
+  def delete_panel(self, panel_path: list[int]):
+    if not panel_path:  # Root deletion
+      old_panel = self.layout["panel"]
+      old_panel.destroy_ui()
+      self.active_panels.remove(old_panel)
+      new_panel = TimeSeriesPanel(self.data_manager, self.playback_manager)
+      self.active_panels.append(new_panel)
+      self.layout = {"type": "panel", "panel": new_panel}
+      self._rebuild_ui_at_path([])
+      return
 
+    parent, child_index = self._get_parent_and_index(panel_path)
+    layout_to_delete = parent["children"][child_index]
+    self._cleanup_ui_recursive(layout_to_delete)
+
+    parent["children"].pop(child_index)
+    parent["proportions"].pop(child_index)
+
+    if len(parent["children"]) == 1: # remove parent and collapse
+      remaining_child = parent["children"][0]
+      if len(panel_path) == 1: # parent is at root level - promote remaining child to root
+        self.layout = remaining_child
+        self._rebuild_ui_at_path([])
+      else: # replace parent with remaining child in grandparent
+        grandparent_path = panel_path[:-2]
+        parent_index = panel_path[-2]
+        self._replace_layout_at_path(grandparent_path + [parent_index], remaining_child)
+        self._rebuild_ui_at_path(grandparent_path + [parent_index])
+    else: # redistribute proportions
+      equal_prop = 1.0 / len(parent["children"])
+      parent["proportions"] = [equal_prop] * len(parent["children"])
+      self._rebuild_ui_at_path(panel_path[:-1])
+
+  def update_all_panels(self):
+    for panel in self.active_panels:
+      panel.update()
+
+  def _get_layout_at_path(self, path: list[int]) -> dict:
+    current = self.layout
+    for index in path:
+      current = current["children"][index]
+    return current
+
+  def _get_parent_and_index(self, path: list[int]) -> tuple:
+    return (None, -1) if not path else (self._get_layout_at_path(path[:-1]), path[-1])
+
+  def _replace_layout_at_path(self, path: list[int], new_layout: dict):
+    if not path:
+      self.layout = new_layout
+    else:
+      parent, index = self._get_parent_and_index(path)
+      parent["children"][index] = new_layout
 
-class SplitterNode(LayoutNode):
-  def __init__(self, children: list[LayoutNode], orientation: str = "horizontal", node_id: str | None = None):
-    super().__init__(node_id)
-    self.children = children if children else []
-    self.orientation = orientation
-    self.child_proportions = [1.0 / len(self.children) for _ in self.children] if self.children else []
-    self.child_container_tags: list[str] = []  # Track container tags for resizing
+  def _path_to_tag(self, path: list[int], prefix: str = "") -> str:
+    path_str = "_".join(map(str, path)) if path else "root"
+    return f"{prefix}_{path_str}" if prefix else path_str
 
-  def add_child(self, child: LayoutNode, index: int = None):
-    if index is None:
-      self.children.append(child)
-      self.child_proportions.append(0.0)
+  def _create_ui_recursive(self, layout: dict, parent_tag: str, path: list[int], width: int, height: int):
+    if layout["type"] == "panel":
+      self._create_panel_ui(layout, parent_tag, path)
     else:
-      self.children.insert(index, child)
-      self.child_proportions.insert(index, 0.0)
-    self._redistribute_proportions()
-
-  def remove_child(self, child: LayoutNode):
-    if child in self.children:
-      index = self.children.index(child)
-      self.children.remove(child)
-      self.child_proportions.pop(index)
-      child.destroy_ui()
-      if self.children:
-        self._redistribute_proportions()
-
-  def replace_child(self, old_child: LayoutNode, new_child: LayoutNode):
-    try:
-      index = self.children.index(old_child)
-      self.children[index] = new_child
-      return index
-    except ValueError:
-      return None
-
-  def _redistribute_proportions(self):
-    if self.children:
-      equal_proportion = 1.0 / len(self.children)
-      self.child_proportions = [equal_proportion for _ in self.children]
-
-  def resize_children(self):
-    if not self.tag or not dpg.does_item_exist(self.tag):
-      return
+      self._create_split_ui(layout, parent_tag, path, width, height)
 
-    available_width, available_height = dpg.get_item_rect_size(dpg.get_item_parent(self.tag))
+  def _create_panel_ui(self, layout: dict, parent_tag: str, path: list[int]):
+    panel_tag = self._path_to_tag(path, "panel")
 
-    for i, container_tag in enumerate(self.child_container_tags):
-      if not dpg.does_item_exist(container_tag):
-        continue
+    with dpg.child_window(tag=panel_tag, parent=parent_tag, border=True, width=-1, height=-1, no_scrollbar=True):
+      with dpg.group(horizontal=True):
+        dpg.add_input_text(default_value=layout["panel"].title, width=int(100 * self.scale), callback=lambda s, v: setattr(layout["panel"], "title", v))
+        dpg.add_combo(items=["Time Series"], default_value="Time Series", width=int(100 * self.scale))
+        dpg.add_button(label="Clear", callback=lambda: self._clear_panel(layout["panel"]), width=int(50 * self.scale))
+        dpg.add_button(label="Delete", callback=lambda: self.delete_panel(path), width=int(50 * self.scale))
+        dpg.add_button(label="Split H", callback=lambda: self.split_panel(path, "horizontal"), width=int(50 * self.scale))
+        dpg.add_button(label="Split V", callback=lambda: self.split_panel(path, "vertical"), width=int(50 * self.scale))
 
-      proportion = self.child_proportions[i] if i < len(self.child_proportions) else (1.0 / len(self.children))
+      dpg.add_separator()
 
-      if self.orientation == "horizontal":
-        new_width = max(100, int(available_width * proportion))
-        dpg.configure_item(container_tag, width=new_width)
-      else:
-        new_height = max(100, int(available_height * proportion))
-        dpg.configure_item(container_tag, height=new_height)
+      content_tag = self._path_to_tag(path, "content")
+      with dpg.child_window(tag=content_tag, border=False, height=-1, width=-1, no_scrollbar=True):
+        layout["panel"].create_ui(content_tag)
 
-      child = self.children[i] if i < len(self.children) else None
-      if child and isinstance(child, SplitterNode):
-        child.resize_children()
+  def _create_split_ui(self, layout: dict, parent_tag: str, path: list[int], width: int, height: int):
+    split_tag = self._path_to_tag(path, "split")
+    is_horizontal = layout["orientation"] == "horizontal"
 
-  def create_ui(self, parent_tag: str, width: int = -1, height: int = -1):
-    self.tag = f"splitter_{self.node_id}"
-    self.child_container_tags = []
+    with dpg.group(tag=split_tag, parent=parent_tag, horizontal=is_horizontal):
+      for i, (child_layout, proportion) in enumerate(zip(layout["children"], layout["proportions"], strict=True)):
+        child_path = path + [i]
+        container_tag = self._path_to_tag(child_path, "container")
 
-    if self.orientation == "horizontal":
-      with dpg.group(tag=self.tag, parent=parent_tag, horizontal=True):
-        for i, child in enumerate(self.children):
-          proportion = self.child_proportions[i]
+        if is_horizontal:
           child_width = max(100, int(width * proportion))
-          container_tag = f"child_container_{self.node_id}_{i}"
-          self.child_container_tags.append(container_tag)
-
-          with dpg.child_window(tag=container_tag, width=child_width, height=-1, border=False, no_scrollbar=True, resizable_x=False):
-            child.create_ui(container_tag, child_width, height)
-    else:
-      with dpg.group(tag=self.tag, parent=parent_tag):
-        for i, child in enumerate(self.children):
-          proportion = self.child_proportions[i]
+          with dpg.child_window(tag=container_tag, width=child_width, height=-1, border=False, no_scrollbar=True):
+            self._create_ui_recursive(child_layout, container_tag, child_path, child_width, height)
+        else:
           child_height = max(100, int(height * proportion))
-          container_tag = f"child_container_{self.node_id}_{i}"
-          self.child_container_tags.append(container_tag)
-
-          with dpg.child_window(tag=container_tag, width=-1, height=child_height, border=False, no_scrollbar=True, resizable_y=False):
-            child.create_ui(container_tag, width, child_height)
+          with dpg.child_window(tag=container_tag, width=-1, height=child_height, border=False, no_scrollbar=True):
+            self._create_ui_recursive(child_layout, container_tag, child_path, width, child_height)
 
-  def destroy_ui(self):
-    for child in self.children:
-      if child:
-        child.destroy_ui()
-    if self.tag and dpg.does_item_exist(self.tag):
-      dpg.delete_item(self.tag)
-    self.child_container_tags.clear()
+  def _rebuild_ui_at_path(self, path: list[int]):
+    layout = self._get_layout_at_path(path)
 
-
-class PlotLayoutManager:
-  def __init__(self, data_manager: DataManager, playback_manager, scale: float = 1.0):
-    self.data_manager = data_manager
-    self.playback_manager = playback_manager
-    self.scale = scale
-    self.container_tag = "plot_layout_container"
-    self._initialize_default_layout()
-
-  def _initialize_default_layout(self):
-    panel = TimeSeriesPanel(self.data_manager, self.playback_manager)
-    self.root_node = LeafNode(panel, layout_manager=self, scale=self.scale)
-
-  def create_ui(self, parent_tag: str):
-    if dpg.does_item_exist(self.container_tag):
-      dpg.delete_item(self.container_tag)
-
-    with dpg.child_window(tag=self.container_tag, parent=parent_tag, border=False, width=-1, height=-1, no_scrollbar=True):
+    if not path:  # Root update
+      dpg.delete_item(self.container_tag, children_only=True)
       container_width, container_height = dpg.get_item_rect_size(self.container_tag)
-      self.root_node.create_ui(self.container_tag, container_width, container_height)
-
-  def on_viewport_resize(self):
-    if isinstance(self.root_node, SplitterNode):
-      self.root_node.resize_children()
-
-  def split_node(self, node: LeafNode, orientation: str):
-    # create new panel for the split
-    new_panel = TimeSeriesPanel(self.data_manager, self.playback_manager)  # TODO: create same type of panel as the split
-    new_leaf = LeafNode(new_panel, layout_manager=self, scale=self.scale)
-
-    parent_node, child_index = self._find_parent_and_index(node)
-
-    if parent_node is None:  # root node - create new splitter as root
-      self.root_node = SplitterNode([node, new_leaf], orientation)
-      self._update_ui_for_node(self.root_node, self.container_tag)
-    elif isinstance(parent_node, SplitterNode) and parent_node.orientation == orientation:  # same orientation - add to existing splitter
-      parent_node.add_child(new_leaf, child_index + 1)
-      self._update_ui_for_node(parent_node)
-    else:  # different orientation - replace node with new splitter
-      new_splitter = SplitterNode([node, new_leaf], orientation)
-      self._replace_child_in_parent(parent_node, node, new_splitter)
-
-  def delete_node(self, node: LeafNode):  # TODO: actually delete the node, not just the ui for the node
-    parent_node, child_index = self._find_parent_and_index(node)
-
-    if parent_node is None:  # root deletion - replace with new default
-      node.destroy_ui()
-      self._initialize_default_layout()
-      self._update_ui_for_node(self.root_node, self.container_tag)
-    elif isinstance(parent_node, SplitterNode):
-      parent_node.remove_child(node)
-      if len(parent_node.children) == 1:  # collapse splitter --> leaf to just leaf
-        remaining_child = parent_node.children[0]
-        grandparent_node, parent_index = self._find_parent_and_index(parent_node)
-
-        if grandparent_node is None:  # promote remaining child to root
-          parent_node.children.remove(remaining_child)
-          self.root_node = remaining_child
-          parent_node.destroy_ui()
-          self._update_ui_for_node(self.root_node, self.container_tag)
-        else:  # replace splitter with remaining child in grandparent node
-          self._replace_child_in_parent(grandparent_node, parent_node, remaining_child)
-      else:  # update splpitter contents
-        self._update_ui_for_node(parent_node)
-
-  def _replace_child_in_parent(self, parent_node: SplitterNode, old_child: LayoutNode, new_child: LayoutNode):
-    child_index = parent_node.children.index(old_child)
-    child_container_tag = f"child_container_{parent_node.node_id}_{child_index}"
-
-    parent_node.replace_child(old_child, new_child)
-
-    # Clean up old child if it's being replaced (not just moved)
-    if old_child != new_child:
-      old_child.destroy_ui()
-
-    if dpg.does_item_exist(child_container_tag):
-      dpg.delete_item(child_container_tag, children_only=True)
-      container_width, container_height = dpg.get_item_rect_size(child_container_tag)
-      new_child.create_ui(child_container_tag, container_width, container_height)
-
-  def _update_ui_for_node(self, node: LayoutNode, container_tag: str = None):
-    if container_tag:  #  update node in a specific container (usually root)
-      dpg.delete_item(container_tag, children_only=True)
-      container_width, container_height = dpg.get_item_rect_size(container_tag)
-      node.create_ui(container_tag, container_width, container_height)
-    else:  # update node in its current location (splitter updates)
-      if node.tag and dpg.does_item_exist(node.tag):
-        parent_container = dpg.get_item_parent(node.tag)
-        node.destroy_ui()
-        if parent_container and dpg.does_item_exist(parent_container):
-          parent_width, parent_height = dpg.get_item_rect_size(parent_container)
-          node.create_ui(parent_container, parent_width, parent_height)
-
-  def _find_parent_and_index(self, target_node: LayoutNode):  # TODO: probably can be stored in child
-    def search_recursive(node: LayoutNode | None, parent: LayoutNode | None = None, index: int = 0):
-      if node == target_node:
-        return parent, index
-      if isinstance(node, SplitterNode):
-        for i, child in enumerate(node.children):
-          result = search_recursive(child, node, i)
-          if result[0] is not None:
-            return result
-      return None, None
-
-    return search_recursive(self.root_node)
+      self._create_ui_recursive(layout, self.container_tag, path, container_width, container_height)
+    else:
+      container_tag = self._path_to_tag(path, "container")
+      if dpg.does_item_exist(container_tag):
+        self._cleanup_ui_recursive(layout)
+        dpg.delete_item(container_tag, children_only=True)
+        width, height = dpg.get_item_rect_size(container_tag)
+        self._create_ui_recursive(layout, container_tag, path, width, height)
+
+  def _cleanup_ui_recursive(self, layout: dict):
+    if layout["type"] == "panel":
+      panel = layout["panel"]
+      panel.destroy_ui()
+      if panel in self.active_panels:
+        self.active_panels.remove(panel)
+    else:
+      for child in layout["children"]:
+        self._cleanup_ui_recursive(child)
+
+  def _resize_splits_recursive(self, layout: dict, path: list[int]):
+    if layout["type"] == "split":
+      split_tag = self._path_to_tag(path, "split")
+      if dpg.does_item_exist(split_tag):
+        parent_tag = dpg.get_item_parent(split_tag)
+        available_width, available_height = dpg.get_item_rect_size(parent_tag)
+
+        for i, proportion in enumerate(layout["proportions"]):
+          child_path = path + [i]
+          container_tag = self._path_to_tag(child_path, "container")
+          if dpg.does_item_exist(container_tag):
+            if layout["orientation"] == "horizontal":
+              dpg.configure_item(container_tag, width=max(100, int(available_width * proportion)))
+            else:
+              dpg.configure_item(container_tag, height=max(100, int(available_height * proportion)))
+
+          self._resize_splits_recursive(layout["children"][i], child_path)
+
+  def _clear_panel(self, panel):
+    if hasattr(panel, 'clear_all_series'):
+      panel.clear_all_series()
diff --git a/tools/jotpluggler/pluggle.py b/tools/jotpluggler/pluggle.py
index a44fa2ed3c..616905a5d7 100755
--- a/tools/jotpluggler/pluggle.py
+++ b/tools/jotpluggler/pluggle.py
@@ -9,7 +9,7 @@ import numpy as np
 from openpilot.common.basedir import BASEDIR
 from openpilot.tools.jotpluggler.data import DataManager
 from openpilot.tools.jotpluggler.views import DataTreeView
-from openpilot.tools.jotpluggler.layout import PlotLayoutManager, SplitterNode, LeafNode
+from openpilot.tools.jotpluggler.layout import PlotLayoutManager
 
 DEMO_ROUTE = "a2a0ccea32023010|2023-07-27--13-01-19"
 
@@ -173,12 +173,12 @@ class MainController:
 
       dpg.set_value("fps_counter", f"{dpg.get_frame_rate():.1f} FPS")
 
-  def _update_visible_set(self): # for some reason, dpg has no way to easily check for visibility, and checking is slow...
+  def _update_visible_set(self):  # for some reason, dpg has no way to easily check for visibility, and checking is slow...
     all_paths = list(self.data_tree_view.created_leaf_paths)
     if not all_paths:
       self.visible_paths.clear()
       return
-    chunk_size = min(50, len(all_paths)) # check up to 50 paths per frame
+    chunk_size = min(50, len(all_paths))  # check up to 50 paths per frame
     end_index = min(self.check_index + chunk_size, len(all_paths))
     for i in range(self.check_index, end_index):
       path = all_paths[i]
@@ -192,7 +192,7 @@ class MainController:
   def _update_data_values(self):
     value_column_width = dpg.get_item_rect_size("data_pool_window")[0] // 2
 
-    for path in self.visible_paths.copy(): # avoid modification during iteration
+    for path in self.visible_paths.copy():  # avoid modification during iteration
       value_tag = f"value_{path}"
       group_tag = f"group_{path}"
 
@@ -207,16 +207,7 @@ class MainController:
         dpg.set_value(value_tag, formatted_value)
 
   def _update_timeline_indicators(self, current_time_s: float):
-    def update_node_recursive(node):
-      if isinstance(node, LeafNode):
-        if hasattr(node.panel, 'update_timeline_indicator'):
-          node.panel.update_timeline_indicator(current_time_s)
-      elif isinstance(node, SplitterNode):
-        for child in node.children:
-          update_node_recursive(child)
-
-    if self.plot_layout_manager.root_node:
-      update_node_recursive(self.plot_layout_manager.root_node)
+    self.plot_layout_manager.update_all_panels()
 
 
 def main(route_to_load=None):
diff --git a/tools/jotpluggler/views.py b/tools/jotpluggler/views.py
index fd9fd09007..94cc4e4174 100644
--- a/tools/jotpluggler/views.py
+++ b/tools/jotpluggler/views.py
@@ -28,6 +28,9 @@ class ViewPanel(ABC):
   def get_panel_type(self) -> str:
     pass
 
+  def update(self):
+    pass
+
 
 class TimeSeriesPanel(ViewPanel):
   def __init__(self, data_manager: DataManager, playback_manager, panel_id: str | None = None):
@@ -61,6 +64,10 @@ class TimeSeriesPanel(ViewPanel):
 
     self._ui_created = True
 
+  def update(self):
+    if self._ui_created:
+      self.update_timeline_indicator(self.playback_manager.current_time_s)
+
   def update_timeline_indicator(self, current_time_s: float):
     if not self._ui_created or not dpg.does_item_exist(self.timeline_indicator_tag):
       return
@@ -149,7 +156,7 @@ class DataTreeView:
     self.ui_lock = ui_lock
     self.current_search = ""
     self.data_tree = DataTreeNode(name="root")
-    self.ui_render_queue: deque[tuple[DataTreeNode, str, str, bool]] = deque() # (node, parent_tag, search_term, is_leaf)
+    self.ui_render_queue: deque[tuple[DataTreeNode, str, str, bool]] = deque()  # (node, parent_tag, search_term, is_leaf)
     self.visible_expanded_nodes: set[str] = set()
     self.created_leaf_paths: set[str] = set()
     self._all_paths_cache: list[str] = []
@@ -176,7 +183,6 @@ class DataTreeView:
     for child in sorted(self.data_tree.children.values(), key=self._natural_sort_key):
       self.ui_render_queue.append((child, "data_tree_container", search_term, child.is_leaf))
 
-
   def _add_paths_to_tree(self, paths, incremental=False):
     search_term = self.current_search.strip().lower()
     filtered_paths = [path for path in paths if self._should_show_path(path, search_term)]
@@ -228,7 +234,7 @@ class DataTreeView:
 
   def update_frame(self):
     items_processed = 0
-    while self.ui_render_queue and items_processed < self.MAX_ITEMS_PER_FRAME: # process up to MAX_ITEMS_PER_FRAME to maintain performance
+    while self.ui_render_queue and items_processed < self.MAX_ITEMS_PER_FRAME:  # process up to MAX_ITEMS_PER_FRAME to maintain performance
       node, parent_tag, search_term, is_leaf = self.ui_render_queue.popleft()
       if is_leaf:
         self._create_leaf_ui(node, parent_tag)
@@ -269,7 +275,7 @@ class DataTreeView:
     node.ui_tag = node_tag
 
     label = f"{node.name} ({node.child_count} fields)"
-    should_open = (bool(search_term) and len(search_term) > 1 and any(search_term in path for path in self._get_descendant_paths(node)))
+    should_open = bool(search_term) and len(search_term) > 1 and any(search_term in path for path in self._get_descendant_paths(node))
 
     with dpg.tree_node(label=label, parent=parent_tag, tag=node_tag, default_open=should_open, open_on_arrow=True, open_on_double_click=True) as tree_node:
       with dpg.item_handler_registry() as handler:

From d1aac22624a3fd2f6619f84f7ad48a726e07b2a7 Mon Sep 17 00:00:00 2001
From: "Quantizr (Jimmy)" <9859727+Quantizr@users.noreply.github.com>
Date: Fri, 29 Aug 2025 21:03:35 -0700
Subject: [PATCH 13/19] resizable layouts

---
 tools/jotpluggler/layout.py | 287 +++++++++++++++++++++++-------------
 tools/jotpluggler/views.py  |   7 +-
 2 files changed, 187 insertions(+), 107 deletions(-)

diff --git a/tools/jotpluggler/layout.py b/tools/jotpluggler/layout.py
index 683bf70a22..74551f92d2 100644
--- a/tools/jotpluggler/layout.py
+++ b/tools/jotpluggler/layout.py
@@ -2,6 +2,9 @@ import dearpygui.dearpygui as dpg
 from openpilot.tools.jotpluggler.data import DataManager
 from openpilot.tools.jotpluggler.views import TimeSeriesPanel
 
+GRIP_SIZE = 4
+MIN_PANE_SIZE = 60
+
 
 class PlotLayoutManager:
   def __init__(self, data_manager: DataManager, playback_manager, scale: float = 1.0):
@@ -9,10 +12,9 @@ class PlotLayoutManager:
     self.playback_manager = playback_manager
     self.scale = scale
     self.container_tag = "plot_layout_container"
-    self.active_panels = []
+    self.active_panels: list = []
 
     initial_panel = TimeSeriesPanel(data_manager, playback_manager)
-    self.active_panels.append(initial_panel)
     self.layout = {"type": "panel", "panel": initial_panel}
 
   def create_ui(self, parent_tag: str):
@@ -23,33 +25,57 @@ class PlotLayoutManager:
       container_width, container_height = dpg.get_item_rect_size(self.container_tag)
       self._create_ui_recursive(self.layout, self.container_tag, [], container_width, container_height)
 
-  def on_viewport_resize(self):
-    self._resize_splits_recursive(self.layout, [])
+  def _create_ui_recursive(self, layout: dict, parent_tag: str, path: list[int], width: int, height: int):
+    if layout["type"] == "panel":
+      self._create_panel_ui(layout, parent_tag, path)
+    else:
+      self._create_split_ui(layout, parent_tag, path, width, height)
 
-  def split_panel(self, panel_path: list[int], orientation: str):
-    current_layout = self._get_layout_at_path(panel_path)
-    existing_panel = current_layout["panel"]
-    new_panel = TimeSeriesPanel(self.data_manager, self.playback_manager)
-    self.active_panels.append(new_panel)
+  def _create_panel_ui(self, layout: dict, parent_tag: str, path: list[int]):
+    panel_tag = self._path_to_tag(path, "panel")
+    panel = layout["panel"]
+    self.active_panels.append(panel)
 
-    parent, child_index = self._get_parent_and_index(panel_path)
+    with dpg.child_window(tag=panel_tag, parent=parent_tag, border=True, width=-1, height=-1, no_scrollbar=True):
+      with dpg.group(horizontal=True):
+        dpg.add_input_text(default_value=panel.title, width=int(100 * self.scale), callback=lambda s, v: setattr(panel, "title", v))
+        dpg.add_combo(items=["Time Series"], default_value="Time Series", width=int(100 * self.scale))
+        dpg.add_button(label="Clear", callback=lambda: self.clear_panel(panel), width=int(40 * self.scale))
+        dpg.add_button(label="Delete", callback=lambda: self.delete_panel(path), width=int(40 * self.scale))
+        dpg.add_button(label="Split H", callback=lambda: self.split_panel(path, 0), width=int(40 * self.scale))
+        dpg.add_button(label="Split V", callback=lambda: self.split_panel(path, 1), width=int(40 * self.scale))
 
-    if parent is None:  # Root split
-      self.layout = {
-        "type": "split",
-        "orientation": orientation,
-        "children": [{"type": "panel", "panel": existing_panel}, {"type": "panel", "panel": new_panel}],
-        "proportions": [0.5, 0.5],
-      }
-      self._rebuild_ui_at_path([])
-    elif parent["type"] == "split" and parent["orientation"] == orientation:
-      parent["children"].insert(child_index + 1, {"type": "panel", "panel": new_panel})
-      parent["proportions"] = [1.0 / len(parent["children"])] * len(parent["children"])
-      self._rebuild_ui_at_path(panel_path[:-1])
-    else:
-      new_split = {"type": "split", "orientation": orientation, "children": [current_layout, {"type": "panel", "panel": new_panel}], "proportions": [0.5, 0.5]}
-      self._replace_layout_at_path(panel_path, new_split)
-      self._rebuild_ui_at_path(panel_path)
+      dpg.add_separator()
+
+      content_tag = self._path_to_tag(path, "content")
+      with dpg.child_window(tag=content_tag, border=False, height=-1, width=-1, no_scrollbar=True):
+        panel.create_ui(content_tag)
+
+  def _create_split_ui(self, layout: dict, parent_tag: str, path: list[int], width: int, height: int):
+    split_tag = self._path_to_tag(path, "split")
+    orientation = layout["orientation"]
+    min_pane_size = int(MIN_PANE_SIZE * self.scale)
+    grip_size = int(GRIP_SIZE * self.scale)
+    num_grips = len(layout["children"]) - 1
+
+    with dpg.group(tag=split_tag, parent=parent_tag, horizontal=orientation == 0):
+      for i, (child_layout, proportion) in enumerate(zip(layout["children"], layout["proportions"], strict=True)):
+        child_path = path + [i]
+        container_tag = self._path_to_tag(child_path, "container")
+
+        size = [width, height]  # pass through since get_item_rect_size is unavailble until rendered
+        fill_size = [-1, -1]  # fill up to the border upon resize
+        calculated_size = max(min_pane_size, int((size[orientation] - (num_grips * grip_size)) * proportion))
+        size[orientation] = fill_size[orientation] = calculated_size
+
+        with dpg.child_window(tag=container_tag, width=fill_size[0], height=fill_size[1], border=False, no_scrollbar=True):
+          self._create_ui_recursive(child_layout, container_tag, child_path, size[0], size[1])
+
+        if i < len(layout["children"]) - 1:  # Add grip between panes (except after the last pane)
+          self._create_grip(split_tag, path, i, orientation)
+
+  def clear_panel(self, panel):
+    panel.clear()
 
   def delete_panel(self, panel_path: list[int]):
     if not panel_path:  # Root deletion
@@ -57,37 +83,113 @@ class PlotLayoutManager:
       old_panel.destroy_ui()
       self.active_panels.remove(old_panel)
       new_panel = TimeSeriesPanel(self.data_manager, self.playback_manager)
-      self.active_panels.append(new_panel)
       self.layout = {"type": "panel", "panel": new_panel}
       self._rebuild_ui_at_path([])
       return
 
     parent, child_index = self._get_parent_and_index(panel_path)
     layout_to_delete = parent["children"][child_index]
-    self._cleanup_ui_recursive(layout_to_delete)
+    self._cleanup_ui_recursive(layout_to_delete, panel_path)
 
     parent["children"].pop(child_index)
     parent["proportions"].pop(child_index)
 
-    if len(parent["children"]) == 1: # remove parent and collapse
+    if len(parent["children"]) == 1:  # remove parent and collapse
       remaining_child = parent["children"][0]
-      if len(panel_path) == 1: # parent is at root level - promote remaining child to root
+      if len(panel_path) == 1:  # parent is at root level - promote remaining child to root
         self.layout = remaining_child
         self._rebuild_ui_at_path([])
-      else: # replace parent with remaining child in grandparent
+      else:  # replace parent with remaining child in grandparent
         grandparent_path = panel_path[:-2]
         parent_index = panel_path[-2]
         self._replace_layout_at_path(grandparent_path + [parent_index], remaining_child)
         self._rebuild_ui_at_path(grandparent_path + [parent_index])
-    else: # redistribute proportions
+    else:  # redistribute proportions
       equal_prop = 1.0 / len(parent["children"])
       parent["proportions"] = [equal_prop] * len(parent["children"])
       self._rebuild_ui_at_path(panel_path[:-1])
 
+  def split_panel(self, panel_path: list[int], orientation: int):
+    current_layout = self._get_layout_at_path(panel_path)
+    existing_panel = current_layout["panel"]
+    new_panel = TimeSeriesPanel(self.data_manager, self.playback_manager)
+    parent, child_index = self._get_parent_and_index(panel_path)
+
+    if parent is None:  # Root split
+      self.layout = {
+        "type": "split",
+        "orientation": orientation,
+        "children": [{"type": "panel", "panel": existing_panel}, {"type": "panel", "panel": new_panel}],
+        "proportions": [0.5, 0.5],
+      }
+      self._rebuild_ui_at_path([])
+    elif parent["type"] == "split" and parent["orientation"] == orientation:  # Same orientation - insert into existing split
+      parent["children"].insert(child_index + 1, {"type": "panel", "panel": new_panel})
+      parent["proportions"] = [1.0 / len(parent["children"])] * len(parent["children"])
+      self._rebuild_ui_at_path(panel_path[:-1])
+    else:  # Different orientation - create new split level
+      new_split = {"type": "split", "orientation": orientation, "children": [current_layout, {"type": "panel", "panel": new_panel}], "proportions": [0.5, 0.5]}
+      self._replace_layout_at_path(panel_path, new_split)
+      self._rebuild_ui_at_path(panel_path)
+
+  def _rebuild_ui_at_path(self, path: list[int]):
+    layout = self._get_layout_at_path(path)
+    if path:
+      container_tag = self._path_to_tag(path, "container")
+    else:  # Root update
+      container_tag = self.container_tag
+
+    self._cleanup_ui_recursive(layout, path)
+    dpg.delete_item(container_tag, children_only=True)
+    width, height = dpg.get_item_rect_size(container_tag)
+    self._create_ui_recursive(layout, container_tag, path, width, height)
+
+  def _cleanup_ui_recursive(self, layout: dict, path: list[int]):
+    if layout["type"] == "panel":
+      panel = layout["panel"]
+      panel.destroy_ui()
+      if panel in self.active_panels:
+        self.active_panels.remove(panel)
+    else:
+      # Clean up grip handler registries for splits BEFORE recursing
+      for i in range(len(layout["children"]) - 1):
+        grip_tag = self._path_to_tag(path, f"grip_{i}")
+        handler_tag = f"{grip_tag}_handler"
+        if dpg.does_item_exist(handler_tag):
+          dpg.delete_item(handler_tag)
+
+      # Recursively cleanup children
+      for i, child in enumerate(layout["children"]):
+        child_path = path + [i]
+        self._cleanup_ui_recursive(child, child_path)
+
   def update_all_panels(self):
     for panel in self.active_panels:
       panel.update()
 
+  def on_viewport_resize(self):
+    self._resize_splits_recursive(self.layout, [])
+
+  def _resize_splits_recursive(self, layout: dict, path: list[int]):
+    if layout["type"] == "split":
+      split_tag = self._path_to_tag(path, "split")
+      if dpg.does_item_exist(split_tag):
+        parent_tag = dpg.get_item_parent(split_tag)
+        grip_size = int(GRIP_SIZE * self.scale)
+        min_pane_size = int(MIN_PANE_SIZE * self.scale)
+        num_grips = len(layout["children"]) - 1
+        orientation = layout["orientation"]
+        available_sizes = dpg.get_item_rect_size(parent_tag)
+        size_properties = ("width", "height")
+
+        for i, proportion in enumerate(layout["proportions"]):
+          child_path = path + [i]
+          container_tag = self._path_to_tag(child_path, "container")
+          if dpg.does_item_exist(container_tag):
+            new_size = max(min_pane_size, int((available_sizes[orientation] - (num_grips * grip_size)) * proportion))
+            dpg.configure_item(container_tag, **{size_properties[orientation]: new_size})
+          self._resize_splits_recursive(layout["children"][i], child_path)
+
   def _get_layout_at_path(self, path: list[int]) -> dict:
     current = self.layout
     for index in path:
@@ -108,91 +210,64 @@ class PlotLayoutManager:
     path_str = "_".join(map(str, path)) if path else "root"
     return f"{prefix}_{path_str}" if prefix else path_str
 
-  def _create_ui_recursive(self, layout: dict, parent_tag: str, path: list[int], width: int, height: int):
-    if layout["type"] == "panel":
-      self._create_panel_ui(layout, parent_tag, path)
-    else:
-      self._create_split_ui(layout, parent_tag, path, width, height)
+  def _create_grip(self, parent_tag: str, path: list[int], grip_index: int, orientation: int):
+    grip_tag = self._path_to_tag(path, f"grip_{grip_index}")
+    grip_size = int(GRIP_SIZE * self.scale)
+    width = grip_size if orientation == 0 else -1
+    height = grip_size if orientation == 1 else -1
 
-  def _create_panel_ui(self, layout: dict, parent_tag: str, path: list[int]):
-    panel_tag = self._path_to_tag(path, "panel")
+    with dpg.child_window(tag=grip_tag, parent=parent_tag, width=width, height=height, no_scrollbar=True, border=False):
+      button_tag = dpg.add_button(label="", width=-1, height=-1)
 
-    with dpg.child_window(tag=panel_tag, parent=parent_tag, border=True, width=-1, height=-1, no_scrollbar=True):
-      with dpg.group(horizontal=True):
-        dpg.add_input_text(default_value=layout["panel"].title, width=int(100 * self.scale), callback=lambda s, v: setattr(layout["panel"], "title", v))
-        dpg.add_combo(items=["Time Series"], default_value="Time Series", width=int(100 * self.scale))
-        dpg.add_button(label="Clear", callback=lambda: self._clear_panel(layout["panel"]), width=int(50 * self.scale))
-        dpg.add_button(label="Delete", callback=lambda: self.delete_panel(path), width=int(50 * self.scale))
-        dpg.add_button(label="Split H", callback=lambda: self.split_panel(path, "horizontal"), width=int(50 * self.scale))
-        dpg.add_button(label="Split V", callback=lambda: self.split_panel(path, "vertical"), width=int(50 * self.scale))
+    with dpg.item_handler_registry(tag=f"{grip_tag}_handler"):
+      user_data = (path, grip_index, orientation)
+      dpg.add_item_active_handler(callback=self._on_grip_drag, user_data=user_data)
+      dpg.add_item_deactivated_handler(callback=self._on_grip_end, user_data=user_data)
+    dpg.bind_item_handler_registry(button_tag, f"{grip_tag}_handler")
 
-      dpg.add_separator()
+  def _on_grip_drag(self, sender, app_data, user_data):
+    path, grip_index, orientation = user_data
+    layout = self._get_layout_at_path(path)
 
-      content_tag = self._path_to_tag(path, "content")
-      with dpg.child_window(tag=content_tag, border=False, height=-1, width=-1, no_scrollbar=True):
-        layout["panel"].create_ui(content_tag)
+    if "_drag_data" not in layout:
+      layout["_drag_data"] = {"initial_proportions": layout["proportions"][:], "start_mouse": dpg.get_mouse_pos(local=False)[orientation]}
+      return
 
-  def _create_split_ui(self, layout: dict, parent_tag: str, path: list[int], width: int, height: int):
-    split_tag = self._path_to_tag(path, "split")
-    is_horizontal = layout["orientation"] == "horizontal"
+    drag_data = layout["_drag_data"]
+    current_coord = dpg.get_mouse_pos(local=False)[orientation]
+    delta = current_coord - drag_data["start_mouse"]
 
-    with dpg.group(tag=split_tag, parent=parent_tag, horizontal=is_horizontal):
-      for i, (child_layout, proportion) in enumerate(zip(layout["children"], layout["proportions"], strict=True)):
-        child_path = path + [i]
-        container_tag = self._path_to_tag(child_path, "container")
+    split_tag = self._path_to_tag(path, "split")
+    if not dpg.does_item_exist(split_tag):
+      return
+    total_size = dpg.get_item_rect_size(split_tag)[orientation]
+    num_grips = len(layout["children"]) - 1
+    usable_size = max(100, total_size - (num_grips * int(GRIP_SIZE * self.scale)))
 
-        if is_horizontal:
-          child_width = max(100, int(width * proportion))
-          with dpg.child_window(tag=container_tag, width=child_width, height=-1, border=False, no_scrollbar=True):
-            self._create_ui_recursive(child_layout, container_tag, child_path, child_width, height)
-        else:
-          child_height = max(100, int(height * proportion))
-          with dpg.child_window(tag=container_tag, width=-1, height=child_height, border=False, no_scrollbar=True):
-            self._create_ui_recursive(child_layout, container_tag, child_path, width, child_height)
+    delta_prop = delta / usable_size
 
-  def _rebuild_ui_at_path(self, path: list[int]):
-    layout = self._get_layout_at_path(path)
+    left_idx = grip_index
+    right_idx = left_idx + 1
+    initial = drag_data["initial_proportions"]
+    min_prop = int(MIN_PANE_SIZE * self.scale) / usable_size
 
-    if not path:  # Root update
-      dpg.delete_item(self.container_tag, children_only=True)
-      container_width, container_height = dpg.get_item_rect_size(self.container_tag)
-      self._create_ui_recursive(layout, self.container_tag, path, container_width, container_height)
-    else:
-      container_tag = self._path_to_tag(path, "container")
-      if dpg.does_item_exist(container_tag):
-        self._cleanup_ui_recursive(layout)
-        dpg.delete_item(container_tag, children_only=True)
-        width, height = dpg.get_item_rect_size(container_tag)
-        self._create_ui_recursive(layout, container_tag, path, width, height)
+    new_left = max(min_prop, initial[left_idx] + delta_prop)
+    new_right = max(min_prop, initial[right_idx] - delta_prop)
 
-  def _cleanup_ui_recursive(self, layout: dict):
-    if layout["type"] == "panel":
-      panel = layout["panel"]
-      panel.destroy_ui()
-      if panel in self.active_panels:
-        self.active_panels.remove(panel)
-    else:
-      for child in layout["children"]:
-        self._cleanup_ui_recursive(child)
+    total_available = initial[left_idx] + initial[right_idx]
+    if new_left + new_right > total_available:
+      if new_left > new_right:
+        new_left = total_available - new_right
+      else:
+        new_right = total_available - new_left
 
-  def _resize_splits_recursive(self, layout: dict, path: list[int]):
-    if layout["type"] == "split":
-      split_tag = self._path_to_tag(path, "split")
-      if dpg.does_item_exist(split_tag):
-        parent_tag = dpg.get_item_parent(split_tag)
-        available_width, available_height = dpg.get_item_rect_size(parent_tag)
+    layout["proportions"] = initial[:]
+    layout["proportions"][left_idx] = new_left
+    layout["proportions"][right_idx] = new_right
 
-        for i, proportion in enumerate(layout["proportions"]):
-          child_path = path + [i]
-          container_tag = self._path_to_tag(child_path, "container")
-          if dpg.does_item_exist(container_tag):
-            if layout["orientation"] == "horizontal":
-              dpg.configure_item(container_tag, width=max(100, int(available_width * proportion)))
-            else:
-              dpg.configure_item(container_tag, height=max(100, int(available_height * proportion)))
+    self._resize_splits_recursive(layout, path)
 
-          self._resize_splits_recursive(layout["children"][i], child_path)
+  def _on_grip_end(self, sender, app_data, user_data):
+    path, _, _ = user_data
+    self._get_layout_at_path(path).pop("_drag_data", None)
 
-  def _clear_panel(self, panel):
-    if hasattr(panel, 'clear_all_series'):
-      panel.clear_all_series()
diff --git a/tools/jotpluggler/views.py b/tools/jotpluggler/views.py
index 94cc4e4174..4309ada88e 100644
--- a/tools/jotpluggler/views.py
+++ b/tools/jotpluggler/views.py
@@ -16,6 +16,10 @@ class ViewPanel(ABC):
     self.panel_id = panel_id or str(uuid.uuid4())
     self.title = "Untitled Panel"
 
+  @abstractmethod
+  def clear(self):
+    pass
+
   @abstractmethod
   def create_ui(self, parent_tag: str):
     pass
@@ -28,6 +32,7 @@ class ViewPanel(ABC):
   def get_panel_type(self) -> str:
     pass
 
+  @abstractmethod
   def update(self):
     pass
 
@@ -117,7 +122,7 @@ class TimeSeriesPanel(ViewPanel):
   def get_panel_type(self) -> str:
     return "timeseries"
 
-  def clear_all_series(self):
+  def clear(self):
     for series_path in list(self._series_data.keys()):
       self.remove_series(series_path)
 

From 9645c979b8a1587e843f6465133b70e7f4196479 Mon Sep 17 00:00:00 2001
From: "Quantizr (Jimmy)" <9859727+Quantizr@users.noreply.github.com>
Date: Sat, 30 Aug 2025 01:26:06 -0700
Subject: [PATCH 14/19] cleanup

---
 tools/jotpluggler/layout.py | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/tools/jotpluggler/layout.py b/tools/jotpluggler/layout.py
index 74551f92d2..a04de407a6 100644
--- a/tools/jotpluggler/layout.py
+++ b/tools/jotpluggler/layout.py
@@ -15,7 +15,7 @@ class PlotLayoutManager:
     self.active_panels: list = []
 
     initial_panel = TimeSeriesPanel(data_manager, playback_manager)
-    self.layout = {"type": "panel", "panel": initial_panel}
+    self.layout: dict = {"type": "panel", "panel": initial_panel}
 
   def create_ui(self, parent_tag: str):
     if dpg.does_item_exist(self.container_tag):
@@ -63,7 +63,7 @@ class PlotLayoutManager:
         child_path = path + [i]
         container_tag = self._path_to_tag(child_path, "container")
 
-        size = [width, height]  # pass through since get_item_rect_size is unavailble until rendered
+        size = [width, height]  # pass through since get_item_rect_size is unavailable until rendered
         fill_size = [-1, -1]  # fill up to the border upon resize
         calculated_size = max(min_pane_size, int((size[orientation] - (num_grips * grip_size)) * proportion))
         size[orientation] = fill_size[orientation] = calculated_size
@@ -151,17 +151,13 @@ class PlotLayoutManager:
       if panel in self.active_panels:
         self.active_panels.remove(panel)
     else:
-      # Clean up grip handler registries for splits BEFORE recursing
       for i in range(len(layout["children"]) - 1):
-        grip_tag = self._path_to_tag(path, f"grip_{i}")
-        handler_tag = f"{grip_tag}_handler"
+        handler_tag = f"{self._path_to_tag(path, f'grip_{i}')}_handler"
         if dpg.does_item_exist(handler_tag):
           dpg.delete_item(handler_tag)
 
-      # Recursively cleanup children
       for i, child in enumerate(layout["children"]):
-        child_path = path + [i]
-        self._cleanup_ui_recursive(child, child_path)
+        self._cleanup_ui_recursive(child, path + [i])
 
   def update_all_panels(self):
     for panel in self.active_panels:
@@ -270,4 +266,3 @@ class PlotLayoutManager:
   def _on_grip_end(self, sender, app_data, user_data):
     path, _, _ = user_data
     self._get_layout_at_path(path).pop("_drag_data", None)
-

From 9414b4a9cb91b1c93f091ba022aef5872b86d706 Mon Sep 17 00:00:00 2001
From: "Quantizr (Jimmy)" <9859727+Quantizr@users.noreply.github.com>
Date: Wed, 3 Sep 2025 15:43:52 -0700
Subject: [PATCH 15/19] downsampling

---
 tools/jotpluggler/data.py    |   8 +-
 tools/jotpluggler/layout.py  |   9 +-
 tools/jotpluggler/pluggle.py |  66 +++++++++++--
 tools/jotpluggler/views.py   | 187 +++++++++++++++++++++++------------
 4 files changed, 194 insertions(+), 76 deletions(-)

diff --git a/tools/jotpluggler/data.py b/tools/jotpluggler/data.py
index 0cbc62bf13..fa91b1d7ae 100644
--- a/tools/jotpluggler/data.py
+++ b/tools/jotpluggler/data.py
@@ -246,6 +246,11 @@ class DataManager:
     with self._lock:
       self._observers.append(callback)
 
+  def remove_observer(self, callback):
+    with self._lock:
+      if callback in self._observers:
+        self._observers.remove(callback)
+
   def _reset(self):
     with self._lock:
       self.loading = True
@@ -261,7 +266,8 @@ class DataManager:
         cloudlog.warning(f"Warning: No log segments found for route: {route}")
         return
 
-      with multiprocessing.Pool() as pool, tqdm.tqdm(total=len(lr.logreader_identifiers), desc="Processing Segments") as pbar:
+      num_processes = max(1, multiprocessing.cpu_count() // 2)
+      with multiprocessing.Pool(processes=num_processes) as pool, tqdm.tqdm(total=len(lr.logreader_identifiers), desc="Processing Segments") as pbar:
         for segment_result, start_time, end_time in pool.imap(_process_segment, lr.logreader_identifiers):
           pbar.update(1)
           if segment_result:
diff --git a/tools/jotpluggler/layout.py b/tools/jotpluggler/layout.py
index a04de407a6..342b5b81e2 100644
--- a/tools/jotpluggler/layout.py
+++ b/tools/jotpluggler/layout.py
@@ -7,14 +7,15 @@ MIN_PANE_SIZE = 60
 
 
 class PlotLayoutManager:
-  def __init__(self, data_manager: DataManager, playback_manager, scale: float = 1.0):
+  def __init__(self, data_manager: DataManager, playback_manager, worker_manager, scale: float = 1.0):
     self.data_manager = data_manager
     self.playback_manager = playback_manager
+    self.worker_manager = worker_manager
     self.scale = scale
     self.container_tag = "plot_layout_container"
     self.active_panels: list = []
 
-    initial_panel = TimeSeriesPanel(data_manager, playback_manager)
+    initial_panel = TimeSeriesPanel(data_manager, playback_manager, worker_manager)
     self.layout: dict = {"type": "panel", "panel": initial_panel}
 
   def create_ui(self, parent_tag: str):
@@ -82,7 +83,7 @@ class PlotLayoutManager:
       old_panel = self.layout["panel"]
       old_panel.destroy_ui()
       self.active_panels.remove(old_panel)
-      new_panel = TimeSeriesPanel(self.data_manager, self.playback_manager)
+      new_panel = TimeSeriesPanel(self.data_manager, self.playback_manager, self.worker_manager)
       self.layout = {"type": "panel", "panel": new_panel}
       self._rebuild_ui_at_path([])
       return
@@ -112,7 +113,7 @@ class PlotLayoutManager:
   def split_panel(self, panel_path: list[int], orientation: int):
     current_layout = self._get_layout_at_path(panel_path)
     existing_panel = current_layout["panel"]
-    new_panel = TimeSeriesPanel(self.data_manager, self.playback_manager)
+    new_panel = TimeSeriesPanel(self.data_manager, self.playback_manager, self.worker_manager)
     parent, child_index = self._get_parent_and_index(panel_path)
 
     if parent is None:  # Root split
diff --git a/tools/jotpluggler/pluggle.py b/tools/jotpluggler/pluggle.py
index 616905a5d7..de8f3dd2b2 100755
--- a/tools/jotpluggler/pluggle.py
+++ b/tools/jotpluggler/pluggle.py
@@ -5,6 +5,9 @@ import pyautogui
 import subprocess
 import dearpygui.dearpygui as dpg
 import threading
+import multiprocessing
+import uuid
+import signal
 import numpy as np
 from openpilot.common.basedir import BASEDIR
 from openpilot.tools.jotpluggler.data import DataManager
@@ -14,6 +17,50 @@ from openpilot.tools.jotpluggler.layout import PlotLayoutManager
 DEMO_ROUTE = "a2a0ccea32023010|2023-07-27--13-01-19"
 
 
+class WorkerManager:
+  def __init__(self, max_workers=None):
+    self.pool = multiprocessing.Pool(max_workers or min(4, multiprocessing.cpu_count()), initializer=WorkerManager.worker_initializer)
+    self.active_tasks = {}
+
+  def submit_task(self, func, args_list, callback=None, task_id=None):
+    task_id = task_id or str(uuid.uuid4())
+
+    if task_id in self.active_tasks:
+      try:
+        self.active_tasks[task_id].terminate()
+      except Exception:
+        pass
+
+    def handle_success(result):
+      self.active_tasks.pop(task_id, None)
+      if callback:
+        try:
+          callback(result)
+        except Exception as e:
+          print(f"Callback for task {task_id} failed: {e}")
+
+    def handle_error(error):
+      self.active_tasks.pop(task_id, None)
+      print(f"Task {task_id} failed: {error}")
+
+    async_result = self.pool.starmap_async(func, args_list, callback=handle_success, error_callback=handle_error)
+    self.active_tasks[task_id] = async_result
+    return task_id
+
+  @staticmethod
+  def worker_initializer():
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+  def shutdown(self):
+    for task in self.active_tasks.values():
+      try:
+        task.terminate()
+      except Exception:
+        pass
+    self.pool.terminate()
+    self.pool.join()
+
+
 class PlaybackManager:
   def __init__(self):
     self.is_playing = False
@@ -62,9 +109,10 @@ class MainController:
     self.scale = scale
     self.data_manager = DataManager()
     self.playback_manager = PlaybackManager()
+    self.worker_manager = WorkerManager()
     self._create_global_themes()
     self.data_tree_view = DataTreeView(self.data_manager, self.ui_lock)
-    self.plot_layout_manager = PlotLayoutManager(self.data_manager, self.playback_manager, scale=self.scale)
+    self.plot_layout_manager = PlotLayoutManager(self.data_manager, self.playback_manager, self.worker_manager, scale=self.scale)
     self.data_manager.add_observer(self.on_data_loaded)
     self.avg_char_width = None
     self.visible_paths: set[str] = set()
@@ -209,6 +257,9 @@ class MainController:
   def _update_timeline_indicators(self, current_time_s: float):
     self.plot_layout_manager.update_all_panels()
 
+  def shutdown(self):
+    self.worker_manager.shutdown()
+
 
 def main(route_to_load=None):
   dpg.create_context()
@@ -241,12 +292,13 @@ def main(route_to_load=None):
   dpg.show_viewport()
 
   # Main loop
-  while dpg.is_dearpygui_running():
-    controller.update_frame(default_font)
-    dpg.render_dearpygui_frame()
-
-  dpg.destroy_context()
-
+  try:
+    while dpg.is_dearpygui_running():
+      controller.update_frame(default_font)
+      dpg.render_dearpygui_frame()
+  finally:
+    controller.shutdown()
+    dpg.destroy_context()
 
 if __name__ == "__main__":
   parser = argparse.ArgumentParser(description="A tool for visualizing openpilot logs.")
diff --git a/tools/jotpluggler/views.py b/tools/jotpluggler/views.py
index 4309ada88e..dbc61ead67 100644
--- a/tools/jotpluggler/views.py
+++ b/tools/jotpluggler/views.py
@@ -1,4 +1,5 @@
 import os
+import queue
 import re
 import uuid
 import threading
@@ -38,25 +39,25 @@ class ViewPanel(ABC):
 
 
 class TimeSeriesPanel(ViewPanel):
-  def __init__(self, data_manager: DataManager, playback_manager, panel_id: str | None = None):
+  def __init__(self, data_manager, playback_manager, worker_manager, panel_id: str | None = None):
     super().__init__(panel_id)
     self.data_manager = data_manager
     self.playback_manager = playback_manager
+    self.worker_manager = worker_manager
     self.title = "Time Series Plot"
-    self.plot_tag: str | None = None
-    self.x_axis_tag: str | None = None
-    self.y_axis_tag: str | None = None
-    self.timeline_indicator_tag: str | None = None
-    self._ui_created = False
-    self._series_data: dict[str, tuple] = {}
-    self.data_manager.add_observer(self.on_data_loaded)
-
-  def create_ui(self, parent_tag: str):
     self.plot_tag = f"plot_{self.panel_id}"
     self.x_axis_tag = f"{self.plot_tag}_x_axis"
     self.y_axis_tag = f"{self.plot_tag}_y_axis"
     self.timeline_indicator_tag = f"{self.plot_tag}_timeline"
+    self._ui_created = False
+    self._series_data = {}
+    self._last_plot_duration = 0
+    self._update_lock = threading.RLock()
+    self.results_queue = queue.Queue()
+    self._new_data = False
 
+  def create_ui(self, parent_tag: str):
+    self.data_manager.add_observer(self.on_data_loaded)
     with dpg.plot(height=-1, width=-1, tag=self.plot_tag, parent=parent_tag, drop_callback=self._on_series_drop, payload_type="TIMESERIES_PAYLOAD"):
       dpg.add_plot_legend()
       dpg.add_plot_axis(dpg.mvXAxis, no_label=True, tag=self.x_axis_tag)
@@ -66,80 +67,138 @@ class TimeSeriesPanel(ViewPanel):
 
     for series_path in list(self._series_data.keys()):
       self.add_series(series_path)
-
     self._ui_created = True
 
   def update(self):
-    if self._ui_created:
-      self.update_timeline_indicator(self.playback_manager.current_time_s)
-
-  def update_timeline_indicator(self, current_time_s: float):
-    if not self._ui_created or not dpg.does_item_exist(self.timeline_indicator_tag):
+    with self._update_lock:
+      if not self._ui_created:
+        return
+
+      if self._new_data:
+        self._new_data = False
+        for series_path in list(self._series_data.keys()):
+          self.add_series(series_path, update=True)
+
+      try:  # check downsample result queue
+        results = self.results_queue.get_nowait()
+        for series_path, downsampled_time, downsampled_values in results:
+          series_tag = f"series_{self.panel_id}_{series_path}"
+          if dpg.does_item_exist(series_tag):
+            dpg.set_value(series_tag, [downsampled_time, downsampled_values])
+      except queue.Empty:
+        pass
+
+      current_time_s = self.playback_manager.current_time_s
+      dpg.set_value(self.timeline_indicator_tag, [[current_time_s], [0]])
+
+      # update timeseries legend label
+      for series_path, (time_array, value_array) in self._series_data.items():
+        position = np.searchsorted(time_array, current_time_s, side='right') - 1
+        if position >= 0 and (current_time_s - time_array[position]) <= 1.0:
+          value = value_array[position]
+          formatted_value = f"{value:.5f}" if np.issubdtype(type(value), np.floating) else str(value)
+          series_tag = f"series_{self.panel_id}_{series_path}"
+          if dpg.does_item_exist(series_tag):
+            dpg.configure_item(series_tag, label=f"{series_path}: {formatted_value}")
+
+      # downsample if plot zoom changed significantly
+      plot_duration = dpg.get_axis_limits(self.x_axis_tag)[1] - dpg.get_axis_limits(self.x_axis_tag)[0]
+      if plot_duration > self._last_plot_duration * 2 or plot_duration < self._last_plot_duration * 0.5:
+        self._downsample_all_series(plot_duration)
+
+  def _downsample_all_series(self, plot_duration):
+    plot_width = dpg.get_item_rect_size(self.plot_tag)[0]
+    if plot_width <= 0 or plot_duration <= 0:
       return
 
-    dpg.set_value(self.timeline_indicator_tag, [[current_time_s], [0]])
-
-    for series_path, (rel_time_array, value_array) in self._series_data.items():
-      position = np.searchsorted(rel_time_array, current_time_s, side='right') - 1
-      value = None
-
-      if position >= 0 and (current_time_s - rel_time_array[position]) <= 1.0:
-        value = value_array[position]
-
-      if value is not None:
-        if np.issubdtype(type(value), np.floating):
-          formatted_value = f"{value:.5f}"
-        else:
-          formatted_value = str(value)
-
-        series_tag = f"series_{self.panel_id}_{series_path}"
-        legend_label = f"{series_path}: {formatted_value}"
-
-        if dpg.does_item_exist(series_tag):
-          dpg.configure_item(series_tag, label=legend_label)
-
-  def add_series(self, series_path: str, update: bool = False) -> bool:
-    if update or series_path not in self._series_data:
-      self._series_data[series_path] = self.data_manager.get_timeseries(series_path)
-
-    rel_time_array, value_array = self._series_data[series_path]
-    series_tag = f"series_{self.panel_id}_{series_path}"
-
-    if dpg.does_item_exist(series_tag):
-      dpg.set_value(series_tag, [rel_time_array, value_array])
-    else:
-      line_series_tag = dpg.add_line_series(x=rel_time_array, y=value_array, label=series_path, parent=self.y_axis_tag, tag=series_tag)
-      dpg.bind_item_theme(line_series_tag, "global_line_theme")
-      dpg.fit_axis_data(self.x_axis_tag)
-      dpg.fit_axis_data(self.y_axis_tag)
-    return True
+    self._last_plot_duration = plot_duration
+    target_points_per_second = plot_width / plot_duration
+    work_items = []
+    for series_path, (time_array, value_array) in self._series_data.items():
+      if len(time_array) == 0:
+        continue
+      series_duration = time_array[-1] - time_array[0] if len(time_array) > 1 else 1
+      points_per_second = len(time_array) / series_duration
+      if points_per_second > target_points_per_second * 2:
+        target_points = max(int(target_points_per_second * series_duration), plot_width)
+        work_items.append((series_path, time_array, value_array, target_points))
+      elif dpg.does_item_exist(f"series_{self.panel_id}_{series_path}"):
+        dpg.set_value(f"series_{self.panel_id}_{series_path}", [time_array, value_array])
+
+    if work_items:
+      self.worker_manager.submit_task(
+        TimeSeriesPanel._downsample_worker, work_items, callback=lambda results: self.results_queue.put(results), task_id=f"downsample_{self.panel_id}"
+      )
+
+  def add_series(self, series_path: str, update: bool = False):
+    with self._update_lock:
+      if update or series_path not in self._series_data:
+        self._series_data[series_path] = self.data_manager.get_timeseries(series_path)
+
+      time_array, value_array = self._series_data[series_path]
+      series_tag = f"series_{self.panel_id}_{series_path}"
+      if dpg.does_item_exist(series_tag):
+        dpg.set_value(series_tag, [time_array, value_array])
+      else:
+        line_series_tag = dpg.add_line_series(x=time_array, y=value_array, label=series_path, parent=self.y_axis_tag, tag=series_tag)
+        dpg.bind_item_theme(line_series_tag, "global_line_theme")
+        dpg.fit_axis_data(self.x_axis_tag)
+        dpg.fit_axis_data(self.y_axis_tag)
+      plot_duration = dpg.get_axis_limits(self.x_axis_tag)[1] - dpg.get_axis_limits(self.x_axis_tag)[0]
+      self._downsample_all_series(plot_duration)
 
   def destroy_ui(self):
-    if self.plot_tag and dpg.does_item_exist(self.plot_tag):
-      dpg.delete_item(self.plot_tag)
-    self._ui_created = False
+    with self._update_lock:
+      self.data_manager.remove_observer(self.on_data_loaded)
+      if dpg.does_item_exist(self.plot_tag):
+        dpg.delete_item(self.plot_tag)
+      self._ui_created = False
 
   def get_panel_type(self) -> str:
     return "timeseries"
 
   def clear(self):
-    for series_path in list(self._series_data.keys()):
-      self.remove_series(series_path)
+    with self._update_lock:
+      for series_path in list(self._series_data.keys()):
+        self.remove_series(series_path)
 
   def remove_series(self, series_path: str):
-    if series_path in self._series_data:
-      series_tag = f"series_{self.panel_id}_{series_path}"
-      if dpg.does_item_exist(series_tag):
-        dpg.delete_item(series_tag)
-      del self._series_data[series_path]
+    with self._update_lock:
+      if series_path in self._series_data:
+        if dpg.does_item_exist(f"series_{self.panel_id}_{series_path}"):
+          dpg.delete_item(f"series_{self.panel_id}_{series_path}")
+        del self._series_data[series_path]
 
   def on_data_loaded(self, data: dict):
-    for series_path in list(self._series_data.keys()):
-      self.add_series(series_path, update=True)
+    self._new_data = True
 
   def _on_series_drop(self, sender, app_data, user_data):
     self.add_series(app_data)
 
+  @staticmethod
+  def _downsample_worker(series_path, time_array, value_array, target_points):
+    if len(time_array) <= target_points:
+      return series_path, time_array, value_array
+
+    step = len(time_array) / target_points
+    indices = []
+
+    for i in range(target_points):
+      start_idx = int(i * step)
+      end_idx = int((i + 1) * step)
+      if start_idx == end_idx:
+        indices.append(start_idx)
+      else:
+        bucket_values = value_array[start_idx:end_idx]
+        min_idx = start_idx + np.argmin(bucket_values)
+        max_idx = start_idx + np.argmax(bucket_values)
+        if min_idx != max_idx:
+          indices.extend([min(min_idx, max_idx), max(min_idx, max_idx)])
+        else:
+          indices.append(min_idx)
+    indices = sorted(set(indices))
+    return series_path, time_array[indices], value_array[indices]
+
 
 class DataTreeNode:
   def __init__(self, name: str, full_path: str = ""):

From de10b64a9c2ee100367f8ea21441ebf7882066b9 Mon Sep 17 00:00:00 2001
From: "Quantizr (Jimmy)" <9859727+Quantizr@users.noreply.github.com>
Date: Wed, 3 Sep 2025 16:31:22 -0700
Subject: [PATCH 16/19] deque for consistency

---
 tools/jotpluggler/views.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/tools/jotpluggler/views.py b/tools/jotpluggler/views.py
index dbc61ead67..c183d2f81b 100644
--- a/tools/jotpluggler/views.py
+++ b/tools/jotpluggler/views.py
@@ -1,5 +1,4 @@
 import os
-import queue
 import re
 import uuid
 import threading
@@ -50,10 +49,10 @@ class TimeSeriesPanel(ViewPanel):
     self.y_axis_tag = f"{self.plot_tag}_y_axis"
     self.timeline_indicator_tag = f"{self.plot_tag}_timeline"
     self._ui_created = False
-    self._series_data = {}
+    self._series_data: dict[str, tuple[list, list]] = {}
     self._last_plot_duration = 0
     self._update_lock = threading.RLock()
-    self.results_queue = queue.Queue()
+    self.results_deque: deque[tuple[str, list, list]] = deque()
     self._new_data = False
 
   def create_ui(self, parent_tag: str):
@@ -74,20 +73,19 @@ class TimeSeriesPanel(ViewPanel):
       if not self._ui_created:
         return
 
-      if self._new_data:
+      if self._new_data:  # handle new data in main thread
         self._new_data = False
         for series_path in list(self._series_data.keys()):
           self.add_series(series_path, update=True)
 
-      try:  # check downsample result queue
-        results = self.results_queue.get_nowait()
+      while self.results_deque:  # handle downsampled results in main thread
+        results = self.results_deque.popleft()
         for series_path, downsampled_time, downsampled_values in results:
           series_tag = f"series_{self.panel_id}_{series_path}"
           if dpg.does_item_exist(series_tag):
             dpg.set_value(series_tag, [downsampled_time, downsampled_values])
-      except queue.Empty:
-        pass
 
+      # update timeline
       current_time_s = self.playback_manager.current_time_s
       dpg.set_value(self.timeline_indicator_tag, [[current_time_s], [0]])
 
@@ -127,7 +125,7 @@ class TimeSeriesPanel(ViewPanel):
 
     if work_items:
       self.worker_manager.submit_task(
-        TimeSeriesPanel._downsample_worker, work_items, callback=lambda results: self.results_queue.put(results), task_id=f"downsample_{self.panel_id}"
+        TimeSeriesPanel._downsample_worker, work_items, callback=lambda results: self.results_deque.append(results), task_id=f"downsample_{self.panel_id}"
       )
 
   def add_series(self, series_path: str, update: bool = False):

From a46445dde42335145b7e414d6a6a13f081efab1b Mon Sep 17 00:00:00 2001
From: "Quantizr (Jimmy)" <9859727+Quantizr@users.noreply.github.com>
Date: Thu, 4 Sep 2025 21:56:05 -0700
Subject: [PATCH 17/19] use item_visible_handler

---
 tools/jotpluggler/pluggle.py |  82 +++------------------------
 tools/jotpluggler/views.py   | 104 ++++++++++++++++++++++-------------
 2 files changed, 75 insertions(+), 111 deletions(-)

diff --git a/tools/jotpluggler/pluggle.py b/tools/jotpluggler/pluggle.py
index de8f3dd2b2..0988f7125c 100755
--- a/tools/jotpluggler/pluggle.py
+++ b/tools/jotpluggler/pluggle.py
@@ -4,11 +4,9 @@ import os
 import pyautogui
 import subprocess
 import dearpygui.dearpygui as dpg
-import threading
 import multiprocessing
 import uuid
 import signal
-import numpy as np
 from openpilot.common.basedir import BASEDIR
 from openpilot.tools.jotpluggler.data import DataManager
 from openpilot.tools.jotpluggler.views import DataTreeView
@@ -88,35 +86,16 @@ class PlaybackManager:
     return self.current_time_s
 
 
-def calculate_avg_char_width(font):
-  sample_text = "abcdefghijklmnopqrstuvwxyz0123456789"
-  if size := dpg.get_text_size(sample_text, font=font):
-    return size[0] / len(sample_text)
-  return None
-
-
-def format_and_truncate(value, available_width: float, avg_char_width: float) -> str:
-  s = f"{value:.5f}" if np.issubdtype(type(value), np.floating) else str(value)
-  max_chars = int(available_width / avg_char_width) - 3
-  if len(s) > max_chars:
-    return s[: max(0, max_chars)] + "..."
-  return s
-
-
 class MainController:
   def __init__(self, scale: float = 1.0):
-    self.ui_lock = threading.Lock()
     self.scale = scale
     self.data_manager = DataManager()
     self.playback_manager = PlaybackManager()
     self.worker_manager = WorkerManager()
     self._create_global_themes()
-    self.data_tree_view = DataTreeView(self.data_manager, self.ui_lock)
+    self.data_tree_view = DataTreeView(self.data_manager, self.playback_manager)
     self.plot_layout_manager = PlotLayoutManager(self.data_manager, self.playback_manager, self.worker_manager, scale=self.scale)
     self.data_manager.add_observer(self.on_data_loaded)
-    self.avg_char_width = None
-    self.visible_paths: set[str] = set()
-    self.check_index = 0
 
   def _create_global_themes(self):
     with dpg.theme(tag="global_line_theme"):
@@ -204,59 +183,16 @@ class MainController:
     dpg.configure_item("play_pause_button", label="Play")
 
   def update_frame(self, font):
-    with self.ui_lock:
-      if self.avg_char_width is None:
-        self.avg_char_width = calculate_avg_char_width(font)
-      self.data_tree_view.update_frame()
-
-      new_time = self.playback_manager.update_time(dpg.get_delta_time())
-      if not dpg.is_item_active("timeline_slider"):
-        dpg.set_value("timeline_slider", new_time)
-
-      self._update_timeline_indicators(new_time)
-
-      if self.avg_char_width:
-        self._update_visible_set()
-        self._update_data_values()
-
-      dpg.set_value("fps_counter", f"{dpg.get_frame_rate():.1f} FPS")
-
-  def _update_visible_set(self):  # for some reason, dpg has no way to easily check for visibility, and checking is slow...
-    all_paths = list(self.data_tree_view.created_leaf_paths)
-    if not all_paths:
-      self.visible_paths.clear()
-      return
-    chunk_size = min(50, len(all_paths))  # check up to 50 paths per frame
-    end_index = min(self.check_index + chunk_size, len(all_paths))
-    for i in range(self.check_index, end_index):
-      path = all_paths[i]
-      value_tag = f"value_{path}"
-      if dpg.does_item_exist(value_tag) and dpg.is_item_visible(value_tag):
-        self.visible_paths.add(path)
-      else:
-        self.visible_paths.discard(path)
-    self.check_index = end_index if end_index < len(all_paths) else 0
-
-  def _update_data_values(self):
-    value_column_width = dpg.get_item_rect_size("data_pool_window")[0] // 2
-
-    for path in self.visible_paths.copy():  # avoid modification during iteration
-      value_tag = f"value_{path}"
-      group_tag = f"group_{path}"
-
-      if not dpg.does_item_exist(value_tag) or not dpg.does_item_exist(group_tag):
-        self.visible_paths.discard(path)
-        continue
-
-      dpg.configure_item(group_tag, xoffset=value_column_width)
-      value = self.data_manager.get_value_at(path, self.playback_manager.current_time_s)
-      if value is not None:
-        formatted_value = format_and_truncate(value, value_column_width, self.avg_char_width)
-        dpg.set_value(value_tag, formatted_value)
-
-  def _update_timeline_indicators(self, current_time_s: float):
+    self.data_tree_view.update_frame(font)
+
+    new_time = self.playback_manager.update_time(dpg.get_delta_time())
+    if not dpg.is_item_active("timeline_slider"):
+      dpg.set_value("timeline_slider", new_time)
+
     self.plot_layout_manager.update_all_panels()
 
+    dpg.set_value("fps_counter", f"{dpg.get_frame_rate():.1f} FPS")
+
   def shutdown(self):
     self.worker_manager.shutdown()
 
diff --git a/tools/jotpluggler/views.py b/tools/jotpluggler/views.py
index c183d2f81b..2cbd9b1335 100644
--- a/tools/jotpluggler/views.py
+++ b/tools/jotpluggler/views.py
@@ -6,7 +6,6 @@ import numpy as np
 from collections import deque
 import dearpygui.dearpygui as dpg
 from abc import ABC, abstractmethod
-from openpilot.tools.jotpluggler.data import DataManager
 
 
 class ViewPanel(ABC):
@@ -213,20 +212,21 @@ class DataTreeNode:
 class DataTreeView:
   MAX_ITEMS_PER_FRAME = 50
 
-  def __init__(self, data_manager: DataManager, ui_lock: threading.Lock):
+  def __init__(self, data_manager, playback_manager):
     self.data_manager = data_manager
-    self.ui_lock = ui_lock
+    self.playback_manager = playback_manager
+    self.lock = threading.RLock()
     self.current_search = ""
     self.data_tree = DataTreeNode(name="root")
-    self.ui_render_queue: deque[tuple[DataTreeNode, str, str, bool]] = deque()  # (node, parent_tag, search_term, is_leaf)
+    self.ui_render_queue: deque[tuple[DataTreeNode, str, str, bool]] = deque()
     self.visible_expanded_nodes: set[str] = set()
-    self.created_leaf_paths: set[str] = set()
     self._all_paths_cache: list[str] = []
     self._previous_paths_set: set[str] = set()
+    self.avg_char_width = None
     self.data_manager.add_observer(self._on_data_loaded)
 
   def _on_data_loaded(self, data: dict):
-    with self.ui_lock:
+    with self.lock:
       if data.get('segment_added'):
         current_paths = set(self.data_manager.get_all_paths())
         new_paths = current_paths - self._previous_paths_set
@@ -253,8 +253,6 @@ class DataTreeView:
     if not filtered_paths:
       return target_tree
 
-    nodes_to_update = set() if incremental else None
-
     for path in sorted(filtered_paths):
       parts = path.split('/')
       current_node = target_tree
@@ -262,20 +260,12 @@ class DataTreeView:
 
       for i, part in enumerate(parts):
         current_path_prefix = f"{current_path_prefix}/{part}" if current_path_prefix else part
-
         if part not in current_node.children:
           current_node.children[part] = DataTreeNode(name=part, full_path=current_path_prefix)
-          if incremental:
-            nodes_to_update.add(current_node)
-
         current_node = current_node.children[part]
-        if incremental and i < len(parts) - 1:
-          nodes_to_update.add(current_node)
 
       if not current_node.is_leaf:
         current_node.is_leaf = True
-        if incremental:
-          nodes_to_update.add(current_node)
 
     self._calculate_child_counts(target_tree)
     if incremental:
@@ -294,27 +284,33 @@ class DataTreeView:
             node = node.children[part]
           self.ui_render_queue.append((node, parent_tag, search_term, True))
 
-  def update_frame(self):
-    items_processed = 0
-    while self.ui_render_queue and items_processed < self.MAX_ITEMS_PER_FRAME:  # process up to MAX_ITEMS_PER_FRAME to maintain performance
-      node, parent_tag, search_term, is_leaf = self.ui_render_queue.popleft()
-      if is_leaf:
-        self._create_leaf_ui(node, parent_tag)
-      else:
-        self._create_node_ui(node, parent_tag, search_term)
-      items_processed += 1
+  def update_frame(self, font):
+    with self.lock:
+      if self.avg_char_width is None and dpg.is_dearpygui_running():
+        self.avg_char_width = self.calculate_avg_char_width(font)
+
+      items_processed = 0
+      while self.ui_render_queue and items_processed < self.MAX_ITEMS_PER_FRAME:
+        node, parent_tag, search_term, is_leaf = self.ui_render_queue.popleft()
+        if is_leaf:
+          self._create_leaf_ui(node, parent_tag)
+        else:
+          self._create_node_ui(node, parent_tag, search_term)
+        items_processed += 1
 
   def search_data(self, search_term: str):
-    self.current_search = search_term
-    self._all_paths_cache = self.data_manager.get_all_paths()
-    self._previous_paths_set = set(self._all_paths_cache)  # Reset tracking after search
-    self._populate_tree()
+    with self.lock:
+      self.current_search = search_term
+      self._all_paths_cache = self.data_manager.get_all_paths()
+      self._previous_paths_set = set(self._all_paths_cache)
+      self._populate_tree()
 
   def _clear_ui(self):
-    dpg.delete_item("data_tree_container", children_only=True)
+    if dpg.does_item_exist("data_tree_container"):
+      dpg.delete_item("data_tree_container", children_only=True)
+
     self.ui_render_queue.clear()
     self.visible_expanded_nodes.clear()
-    self.created_leaf_paths.clear()
 
   def _calculate_child_counts(self, node: DataTreeNode):
     if node.is_leaf:
@@ -342,7 +338,7 @@ class DataTreeView:
     with dpg.tree_node(label=label, parent=parent_tag, tag=node_tag, default_open=should_open, open_on_arrow=True, open_on_double_click=True) as tree_node:
       with dpg.item_handler_registry() as handler:
         dpg.add_item_toggled_open_handler(callback=lambda s, d, u: self._on_node_expanded(node, search_term))
-      dpg.bind_item_handler_registry(tree_node, handler)
+        dpg.bind_item_handler_registry(tree_node, handler)
 
     node.ui_created = True
 
@@ -357,17 +353,34 @@ class DataTreeView:
     with dpg.group(parent=parent_tag, horizontal=True, xoffset=half_split_size, tag=f"group_{node.full_path}") as draggable_group:
       dpg.add_text(node.name)
       dpg.add_text("N/A", tag=f"value_{node.full_path}")
-
       if node.is_plottable_cached is None:
         node.is_plottable_cached = self.data_manager.is_plottable(node.full_path)
-
       if node.is_plottable_cached:
         with dpg.drag_payload(parent=draggable_group, drag_data=node.full_path, payload_type="TIMESERIES_PAYLOAD"):
           dpg.add_text(f"Plot: {node.full_path}")
 
+      with dpg.item_handler_registry() as handler:
+        dpg.add_item_visible_handler(callback=self._on_item_visible, user_data=node.full_path)
+        dpg.bind_item_handler_registry(draggable_group, handler)
+
     node.ui_created = True
     node.ui_tag = f"value_{node.full_path}"
-    self.created_leaf_paths.add(node.full_path)
+
+  def _on_item_visible(self, sender, app_data, user_data):
+    path = user_data
+    if not path or not self.avg_char_width:
+      return
+
+    value_tag = f"value_{path}"
+    value_column_width = dpg.get_item_rect_size("data_pool_window")[0] // 2
+    dpg.configure_item(f"group_{path}", xoffset=value_column_width)
+
+    value = self.data_manager.get_value_at(path, self.playback_manager.current_time_s)
+    if value is not None:
+      formatted_value = self.format_and_truncate(value, value_column_width, self.avg_char_width)
+      dpg.set_value(value_tag, formatted_value)
+    else:
+      dpg.set_value(value_tag, "N/A")
 
   def _queue_children(self, node: DataTreeNode, parent_tag: str, search_term: str):
     for child in sorted(node.children.values(), key=self._natural_sort_key):
@@ -390,10 +403,10 @@ class DataTreeView:
 
   def _remove_children_from_queue(self, collapsed_node_path: str):
     new_queue: deque[tuple] = deque()
-    for node, parent_tag, search_term, is_leaf in self.ui_render_queue:
-      # Keep items that are not children of the collapsed node
+    for item in self.ui_render_queue:
+      node = item[0]
       if not node.full_path.startswith(collapsed_node_path + "/"):
-        new_queue.append((node, parent_tag, search_term, is_leaf))
+        new_queue.append(item)
     self.ui_render_queue = new_queue
 
   def _should_show_path(self, path: str, search_term: str) -> bool:
@@ -414,3 +427,18 @@ class DataTreeView:
       else:
         for path in self._get_descendant_paths(child_node):
           yield f"{child_name_lower}/{path}"
+
+  @staticmethod
+  def calculate_avg_char_width(font):
+    sample_text = "abcdefghijklmnopqrstuvwxyz0123456789"
+    if size := dpg.get_text_size(sample_text, font=font):
+      return size[0] / len(sample_text)
+    return 10.0
+
+  @staticmethod
+  def format_and_truncate(value, available_width: float, avg_char_width: float) -> str:
+    s = f"{value:.5f}" if np.issubdtype(type(value), np.floating) else str(value)
+    max_chars = int(available_width / avg_char_width) - 3
+    if len(s) > max_chars:
+      return s[: max(0, max_chars)] + "..."
+    return s

From 603927014a4e89350cb18d81c891c1ba29c1d01e Mon Sep 17 00:00:00 2001
From: "Quantizr (Jimmy)" <9859727+Quantizr@users.noreply.github.com>
Date: Fri, 5 Sep 2025 18:26:44 -0700
Subject: [PATCH 18/19] only build visible UI

---
 tools/jotpluggler/pluggle.py |  13 +--
 tools/jotpluggler/views.py   | 221 ++++++++++++++++++-----------------
 2 files changed, 114 insertions(+), 120 deletions(-)

diff --git a/tools/jotpluggler/pluggle.py b/tools/jotpluggler/pluggle.py
index 0988f7125c..9244ef9a69 100755
--- a/tools/jotpluggler/pluggle.py
+++ b/tools/jotpluggler/pluggle.py
@@ -129,18 +129,13 @@ class MainController:
     with dpg.window(tag="Primary Window"):
       with dpg.group(horizontal=True):
         # Left panel - Data tree
-        with dpg.child_window(label="Data Pool", width=300 * self.scale, tag="data_pool_window", border=True, resizable_x=True):
+        with dpg.child_window(label="Sidebar", width=300 * self.scale, tag="sidebar_window", border=True, resizable_x=True):
           with dpg.group(horizontal=True):
             dpg.add_input_text(tag="route_input", width=-75 * self.scale, hint="Enter route name...")
             dpg.add_button(label="Load", callback=self.load_route, tag="load_button", width=-1)
           dpg.add_text("Ready to load route", tag="load_status")
           dpg.add_separator()
-          dpg.add_text("Available Data")
-          dpg.add_separator()
-          dpg.add_input_text(tag="search_input", width=-1, hint="Search fields...", callback=self.search_data)
-          dpg.add_separator()
-          with dpg.group(tag="data_tree_container", track_offset=True):
-            pass
+          self.data_tree_view.create_ui("sidebar_window")
 
         # Right panel - Plots and timeline
         with dpg.group():
@@ -169,10 +164,6 @@ class MainController:
       dpg.configure_item("load_button", enabled=False)
       self.data_manager.load_route(route_name)
 
-  def search_data(self):
-    search_term = dpg.get_value("search_input")
-    self.data_tree_view.search_data(search_term)
-
   def toggle_play_pause(self, sender):
     self.playback_manager.toggle_play_pause()
     label = "Pause" if self.playback_manager.is_playing else "Play"
diff --git a/tools/jotpluggler/views.py b/tools/jotpluggler/views.py
index 2cbd9b1335..7c143fc9af 100644
--- a/tools/jotpluggler/views.py
+++ b/tools/jotpluggler/views.py
@@ -206,44 +206,44 @@ class DataTreeNode:
     self.child_count = 0
     self.is_plottable_cached: bool | None = None
     self.ui_created = False
+    self.children_ui_created = False
     self.ui_tag: str | None = None
 
 
 class DataTreeView:
-  MAX_ITEMS_PER_FRAME = 50
+  MAX_NODES_PER_FRAME = 50
 
   def __init__(self, data_manager, playback_manager):
     self.data_manager = data_manager
     self.playback_manager = playback_manager
-    self.lock = threading.RLock()
     self.current_search = ""
     self.data_tree = DataTreeNode(name="root")
-    self.ui_render_queue: deque[tuple[DataTreeNode, str, str, bool]] = deque()
-    self.visible_expanded_nodes: set[str] = set()
-    self._all_paths_cache: list[str] = []
-    self._previous_paths_set: set[str] = set()
+    self.build_queue: deque[tuple[DataTreeNode, str | None, str | int]] = deque()
+    self._all_paths_cache: set[str] = set()
+    self._item_handlers: set[str] = set()
     self.avg_char_width = None
     self.data_manager.add_observer(self._on_data_loaded)
+    self.queued_search = None
+    self.new_data = False
+
+  def create_ui(self, parent_tag: str):
+    with dpg.child_window(parent=parent_tag, border=False, width=-1, height=-1):
+      dpg.add_text("Available Data")
+      dpg.add_separator()
+      dpg.add_input_text(tag="search_input", width=-1, hint="Search fields...", callback=self.search_data)
+      dpg.add_separator()
+      with dpg.group(tag="data_tree_container", track_offset=True):
+        pass
 
   def _on_data_loaded(self, data: dict):
-    with self.lock:
-      if data.get('segment_added'):
-        current_paths = set(self.data_manager.get_all_paths())
-        new_paths = current_paths - self._previous_paths_set
-        if new_paths:
-          self._all_paths_cache = list(current_paths)
-          if not self._previous_paths_set:
-            self._populate_tree()
-          else:
-            self._add_paths_to_tree(new_paths, incremental=True)
-          self._previous_paths_set = current_paths.copy()
+    if data.get('segment_added'):
+      self.new_data = True
 
   def _populate_tree(self):
     self._clear_ui()
-    search_term = self.current_search.strip().lower()
     self.data_tree = self._add_paths_to_tree(self._all_paths_cache, incremental=False)
-    for child in sorted(self.data_tree.children.values(), key=self._natural_sort_key):
-      self.ui_render_queue.append((child, "data_tree_container", search_term, child.is_leaf))
+    if self.data_tree:
+      self._request_children_build(self.data_tree)
 
   def _add_paths_to_tree(self, paths, incremental=False):
     search_term = self.current_search.strip().lower()
@@ -253,6 +253,7 @@ class DataTreeView:
     if not filtered_paths:
       return target_tree
 
+    parent_nodes_to_recheck = set()
     for path in sorted(filtered_paths):
       parts = path.split('/')
       current_node = target_tree
@@ -260,6 +261,8 @@ class DataTreeView:
 
       for i, part in enumerate(parts):
         current_path_prefix = f"{current_path_prefix}/{part}" if current_path_prefix else part
+        if i < len(parts) - 1:
+          parent_nodes_to_recheck.add(current_node)  # for incremental changes from new data
         if part not in current_node.children:
           current_node.children[part] = DataTreeNode(name=part, full_path=current_path_prefix)
         current_node = current_node.children[part]
@@ -269,48 +272,55 @@ class DataTreeView:
 
     self._calculate_child_counts(target_tree)
     if incremental:
-      self._queue_new_ui_items(filtered_paths, search_term)
+      for p_node in parent_nodes_to_recheck:
+        p_node.children_ui_created = False
+        self._request_children_build(p_node)
     return target_tree
 
-  def _queue_new_ui_items(self, new_paths, search_term):
-    for path in new_paths:
-      parts = path.split('/')
-      parent_path = '/'.join(parts[:-1]) if len(parts) > 1 else ""
-      if parent_path == "" or parent_path in self.visible_expanded_nodes:
-        parent_tag = "data_tree_container" if parent_path == "" else f"tree_{parent_path}"
-        if dpg.does_item_exist(parent_tag):
-          node = self.data_tree
-          for part in parts:
-            node = node.children[part]
-          self.ui_render_queue.append((node, parent_tag, search_term, True))
-
   def update_frame(self, font):
-    with self.lock:
-      if self.avg_char_width is None and dpg.is_dearpygui_running():
-        self.avg_char_width = self.calculate_avg_char_width(font)
-
-      items_processed = 0
-      while self.ui_render_queue and items_processed < self.MAX_ITEMS_PER_FRAME:
-        node, parent_tag, search_term, is_leaf = self.ui_render_queue.popleft()
-        if is_leaf:
-          self._create_leaf_ui(node, parent_tag)
+    if self.avg_char_width is None and dpg.is_dearpygui_running():
+      self.avg_char_width = self.calculate_avg_char_width(font)
+
+    if self.new_data:
+      current_paths = set(self.data_manager.get_all_paths())
+      new_paths = current_paths - self._all_paths_cache
+      if new_paths:
+        all_paths_empty = not self._all_paths_cache
+        self._all_paths_cache = current_paths
+        if all_paths_empty:
+          self._populate_tree()
         else:
-          self._create_node_ui(node, parent_tag, search_term)
-        items_processed += 1
-
-  def search_data(self, search_term: str):
-    with self.lock:
-      self.current_search = search_term
-      self._all_paths_cache = self.data_manager.get_all_paths()
-      self._previous_paths_set = set(self._all_paths_cache)
+          self._add_paths_to_tree(new_paths, incremental=True)
+      self.new_data = False
+
+    if self.queued_search is not None:
+      self.current_search = self.queued_search
+      self._all_paths_cache = set(self.data_manager.get_all_paths())
       self._populate_tree()
+      self.queued_search = None
+
+    nodes_processed = 0
+    while self.build_queue and nodes_processed < self.MAX_NODES_PER_FRAME:
+      child_node, parent_tag, before_tag = self.build_queue.popleft()
+      if not child_node.ui_created:
+        if child_node.is_leaf:
+          self._create_leaf_ui(child_node, parent_tag, before_tag)
+        else:
+          self._create_tree_node_ui(child_node, parent_tag, before_tag)
+      nodes_processed += 1
+
+  def search_data(self):
+    self.queued_search = dpg.get_value("search_input")
 
   def _clear_ui(self):
+    for handler_tag in self._item_handlers:
+      dpg.delete_item(handler_tag)
+    self._item_handlers.clear()
+
     if dpg.does_item_exist("data_tree_container"):
       dpg.delete_item("data_tree_container", children_only=True)
 
-    self.ui_render_queue.clear()
-    self.visible_expanded_nodes.clear()
+    self.build_queue.clear()
 
   def _calculate_child_counts(self, node: DataTreeNode):
     if node.is_leaf:
@@ -320,37 +330,34 @@ class DataTreeView:
       for child in node.children.values():
         self._calculate_child_counts(child)
 
-  def _create_node_ui(self, node: DataTreeNode, parent_tag: str, search_term: str):
-    if node.is_leaf:
-      self._create_leaf_ui(node, parent_tag)
-    else:
-      self._create_tree_node_ui(node, parent_tag, search_term)
-
-  def _create_tree_node_ui(self, node: DataTreeNode, parent_tag: str, search_term: str):
-    if not dpg.does_item_exist(parent_tag):
-      return
-    node_tag = f"tree_{node.full_path}"
-    node.ui_tag = node_tag
+  def _create_tree_node_ui(self, node: DataTreeNode, parent_tag: str, before: str | int):
+    tag = f"tree_{node.full_path}"
+    node.ui_tag = tag
+    handler_tag = f"handler_tree_{node.full_path}"
+    self._item_handlers.add(handler_tag)
+    if dpg.does_item_exist(handler_tag):
+      dpg.delete_item(handler_tag)
 
     label = f"{node.name} ({node.child_count} fields)"
+    search_term = self.current_search.strip().lower()
     should_open = bool(search_term) and len(search_term) > 1 and any(search_term in path for path in self._get_descendant_paths(node))
 
-    with dpg.tree_node(label=label, parent=parent_tag, tag=node_tag, default_open=should_open, open_on_arrow=True, open_on_double_click=True) as tree_node:
-      with dpg.item_handler_registry() as handler:
-        dpg.add_item_toggled_open_handler(callback=lambda s, d, u: self._on_node_expanded(node, search_term))
-        dpg.bind_item_handler_registry(tree_node, handler)
+    with dpg.tree_node(label=label, parent=parent_tag, tag=tag, default_open=should_open, open_on_arrow=True, open_on_double_click=True, before=before):
+      with dpg.item_handler_registry(tag=handler_tag):
+        dpg.add_item_toggled_open_handler(callback=lambda s, a, u: self._request_children_build(node, handler_tag))
+        dpg.add_item_visible_handler(callback=lambda s, a, u: self._request_children_build(node, handler_tag))
+      dpg.bind_item_handler_registry(tag, dpg.last_container())
 
     node.ui_created = True
 
-    if should_open:
-      self.visible_expanded_nodes.add(node.full_path)
-      self._queue_children(node, node_tag, search_term)
+  def _create_leaf_ui(self, node: DataTreeNode, parent_tag: str, before: str | int):
+    half_split_size = dpg.get_item_rect_size("sidebar_window")[0] // 2
+    handler_tag = f"handler_leaf_{node.full_path}"
+    self._item_handlers.add(handler_tag)
+    if dpg.does_item_exist(handler_tag):
+      dpg.delete_item(handler_tag)
 
-  def _create_leaf_ui(self, node: DataTreeNode, parent_tag: str):
-    if not dpg.does_item_exist(parent_tag):
-      return
-    half_split_size = dpg.get_item_rect_size("data_pool_window")[0] // 2
-    with dpg.group(parent=parent_tag, horizontal=True, xoffset=half_split_size, tag=f"group_{node.full_path}") as draggable_group:
+    with dpg.group(parent=parent_tag, horizontal=True, xoffset=half_split_size, tag=f"group_{node.full_path}", before=before) as draggable_group:
       dpg.add_text(node.name)
       dpg.add_text("N/A", tag=f"value_{node.full_path}")
       if node.is_plottable_cached is None:
@@ -359,21 +366,23 @@ class DataTreeView:
         with dpg.drag_payload(parent=draggable_group, drag_data=node.full_path, payload_type="TIMESERIES_PAYLOAD"):
           dpg.add_text(f"Plot: {node.full_path}")
 
-      with dpg.item_handler_registry() as handler:
-        dpg.add_item_visible_handler(callback=self._on_item_visible, user_data=node.full_path)
-        dpg.bind_item_handler_registry(draggable_group, handler)
+    with dpg.item_handler_registry(tag=handler_tag):
+      dpg.add_item_visible_handler(callback=self._on_item_visible, user_data=node.full_path)
+    dpg.bind_item_handler_registry(draggable_group, dpg.last_container())
 
     node.ui_created = True
     node.ui_tag = f"value_{node.full_path}"
 
   def _on_item_visible(self, sender, app_data, user_data):
     path = user_data
-    if not path or not self.avg_char_width:
+    group_tag = f"group_{path}"
+    value_tag = f"value_{path}"
+
+    if not self.avg_char_width or not dpg.does_item_exist(group_tag) or not dpg.does_item_exist(value_tag):
       return
 
-    value_tag = f"value_{path}"
-    value_column_width = dpg.get_item_rect_size("data_pool_window")[0] // 2
-    dpg.configure_item(f"group_{path}", xoffset=value_column_width)
+    value_column_width = dpg.get_item_rect_size("sidebar_window")[0] // 2
+    dpg.configure_item(group_tag, xoffset=value_column_width)
 
     value = self.data_manager.get_value_at(path, self.playback_manager.current_time_s)
     if value is not None:
@@ -382,32 +391,26 @@ class DataTreeView:
     else:
       dpg.set_value(value_tag, "N/A")
 
-  def _queue_children(self, node: DataTreeNode, parent_tag: str, search_term: str):
-    for child in sorted(node.children.values(), key=self._natural_sort_key):
-      self.ui_render_queue.append((child, parent_tag, search_term, child.is_leaf))
-
-  def _on_node_expanded(self, node: DataTreeNode, search_term: str):
-    node_tag = f"tree_{node.full_path}"
-    if not dpg.does_item_exist(node_tag):
-      return
-
-    is_expanded = dpg.get_value(node_tag)
-
-    if is_expanded:
-      if node.full_path not in self.visible_expanded_nodes:
-        self.visible_expanded_nodes.add(node.full_path)
-        self._queue_children(node, node_tag, search_term)
-    else:
-      self.visible_expanded_nodes.discard(node.full_path)
-      self._remove_children_from_queue(node.full_path)
-
-  def _remove_children_from_queue(self, collapsed_node_path: str):
-    new_queue: deque[tuple] = deque()
-    for item in self.ui_render_queue:
-      node = item[0]
-      if not node.full_path.startswith(collapsed_node_path + "/"):
-        new_queue.append(item)
-    self.ui_render_queue = new_queue
+  def _request_children_build(self, node: DataTreeNode, handler_tag=None):
+    if not node.children_ui_created and (node.name == "root" or (node.ui_tag is not None and dpg.get_value(node.ui_tag))):  # check root or node expanded
+      if handler_tag and dpg.does_item_exist(handler_tag):
+        dpg.delete_item(handler_tag)
+        self._item_handlers.discard(handler_tag)
+      parent_tag = "data_tree_container" if node.name == "root" else node.ui_tag
+      sorted_children = sorted(node.children.values(), key=self._natural_sort_key)
+
+      for i, child_node in enumerate(sorted_children):
+        if not child_node.ui_created:
+          before_tag: int | str = 0
+          for j in range(i + 1, len(sorted_children)):  # when incrementally building get "before_tag" for correct ordering
+            next_child = sorted_children[j]
+            if next_child.ui_created:
+              candidate_tag = f"group_{next_child.full_path}" if next_child.is_leaf else f"tree_{next_child.full_path}"
+              if dpg.does_item_exist(candidate_tag):
+                before_tag = candidate_tag
+                break
+          self.build_queue.append((child_node, parent_tag, before_tag))
+      node.children_ui_created = True
 
   def _should_show_path(self, path: str, search_term: str) -> bool:
     if 'DEPRECATED' in path and not os.environ.get('SHOW_DEPRECATED'):
@@ -433,7 +436,7 @@ class DataTreeView:
     sample_text = "abcdefghijklmnopqrstuvwxyz0123456789"
     if size := dpg.get_text_size(sample_text, font=font):
       return size[0] / len(sample_text)
-    return 10.0
+    return None
 
   @staticmethod
   def format_and_truncate(value, available_width: float, avg_char_width: float) -> str:

From 0b6c85c89bca7328244901b1e76bcb93e5396722 Mon Sep 17 00:00:00 2001
From: "Quantizr (Jimmy)" <9859727+Quantizr@users.noreply.github.com>
Date: Sat, 6 Sep 2025 00:34:51 -0700
Subject: [PATCH 19/19] don't delete item handlers, add locks, don't expand
 large lists

---
 tools/jotpluggler/views.py | 137 +++++++++++++++++++------------------
 1 file changed, 69 insertions(+), 68 deletions(-)

diff --git a/tools/jotpluggler/views.py b/tools/jotpluggler/views.py
index 7c143fc9af..f3b75a2de1 100644
--- a/tools/jotpluggler/views.py
+++ b/tools/jotpluggler/views.py
@@ -198,9 +198,10 @@ class TimeSeriesPanel(ViewPanel):
 
 
 class DataTreeNode:
-  def __init__(self, name: str, full_path: str = ""):
+  def __init__(self, name: str, full_path: str = "", parent = None):
     self.name = name
     self.full_path = full_path
+    self.parent = parent
     self.children: dict[str, DataTreeNode] = {}
     self.is_leaf = False
     self.child_count = 0
@@ -225,6 +226,7 @@ class DataTreeView:
     self.data_manager.add_observer(self._on_data_loaded)
     self.queued_search = None
     self.new_data = False
+    self._ui_lock = threading.RLock()
 
   def create_ui(self, parent_tag: str):
     with dpg.child_window(parent=parent_tag, border=False, width=-1, height=-1):
@@ -264,7 +266,7 @@ class DataTreeView:
         if i < len(parts) - 1:
           parent_nodes_to_recheck.add(current_node)  # for incremental changes from new data
         if part not in current_node.children:
-          current_node.children[part] = DataTreeNode(name=part, full_path=current_path_prefix)
+          current_node.children[part] = DataTreeNode(name=part, full_path=current_path_prefix, parent=current_node)
         current_node = current_node.children[part]
 
       if not current_node.is_leaf:
@@ -278,45 +280,42 @@ class DataTreeView:
     return target_tree
 
   def update_frame(self, font):
-    if self.avg_char_width is None and dpg.is_dearpygui_running():
-      self.avg_char_width = self.calculate_avg_char_width(font)
-
-    if self.new_data:
-      current_paths = set(self.data_manager.get_all_paths())
-      new_paths = current_paths - self._all_paths_cache
-      if new_paths:
-        all_paths_empty = not self._all_paths_cache
-        self._all_paths_cache = current_paths
-        if all_paths_empty:
-          self._populate_tree()
-        else:
-          self._add_paths_to_tree(new_paths, incremental=True)
-      self.new_data = False
-
-    if self.queued_search is not None:
-      self.current_search = self.queued_search
-      self._all_paths_cache = set(self.data_manager.get_all_paths())
-      self._populate_tree()
-      self.queued_search = None
-
-    nodes_processed = 0
-    while self.build_queue and nodes_processed < self.MAX_NODES_PER_FRAME:
-      child_node, parent_tag, before_tag = self.build_queue.popleft()
-      if not child_node.ui_created:
-        if child_node.is_leaf:
-          self._create_leaf_ui(child_node, parent_tag, before_tag)
-        else:
-          self._create_tree_node_ui(child_node, parent_tag, before_tag)
-      nodes_processed += 1
+    with self._ui_lock:
+      if self.avg_char_width is None and dpg.is_dearpygui_running():
+        self.avg_char_width = self.calculate_avg_char_width(font)
+
+      if self.new_data:
+        current_paths = set(self.data_manager.get_all_paths())
+        new_paths = current_paths - self._all_paths_cache
+        if new_paths:
+          all_paths_empty = not self._all_paths_cache
+          self._all_paths_cache = current_paths
+          if all_paths_empty:
+            self._populate_tree()
+          else:
+            self._add_paths_to_tree(new_paths, incremental=True)
+        self.new_data = False
+
+      if self.queued_search is not None:
+        self.current_search = self.queued_search
+        self._all_paths_cache = set(self.data_manager.get_all_paths())
+        self._populate_tree()
+        self.queued_search = None
+
+      nodes_processed = 0
+      while self.build_queue and nodes_processed < self.MAX_NODES_PER_FRAME:
+        child_node, parent_tag, before_tag = self.build_queue.popleft()
+        if not child_node.ui_created:
+          if child_node.is_leaf:
+            self._create_leaf_ui(child_node, parent_tag, before_tag)
+          else:
+            self._create_tree_node_ui(child_node, parent_tag, before_tag)
+        nodes_processed += 1
 
   def search_data(self):
     self.queued_search = dpg.get_value("search_input")
 
   def _clear_ui(self):
-    for handler_tag in self._item_handlers:
-      dpg.delete_item(handler_tag)
-    self._item_handlers.clear()
-
     if dpg.does_item_exist("data_tree_container"):
       dpg.delete_item("data_tree_container", children_only=True)
 
@@ -341,6 +340,9 @@ class DataTreeView:
     label = f"{node.name} ({node.child_count} fields)"
     search_term = self.current_search.strip().lower()
     should_open = bool(search_term) and len(search_term) > 1 and any(search_term in path for path in self._get_descendant_paths(node))
+    if should_open and node.parent and node.parent.child_count > 100 and node.child_count > 2:
+      label += " (+)"
+      should_open = False
 
     with dpg.tree_node(label=label, parent=parent_tag, tag=tag, default_open=should_open, open_on_arrow=True, open_on_double_click=True, before=before):
       with dpg.item_handler_registry(tag=handler_tag):
@@ -374,43 +376,42 @@ class DataTreeView:
     node.ui_tag = f"value_{node.full_path}"
 
   def _on_item_visible(self, sender, app_data, user_data):
-    path = user_data
-    group_tag = f"group_{path}"
-    value_tag = f"value_{path}"
+    with self._ui_lock:
+      path = user_data
+      group_tag = f"group_{path}"
+      value_tag = f"value_{path}"
 
-    if not self.avg_char_width or not dpg.does_item_exist(group_tag) or not dpg.does_item_exist(value_tag):
-      return
+      if not self.avg_char_width or not dpg.does_item_exist(group_tag) or not dpg.does_item_exist(value_tag):
+        return
 
-    value_column_width = dpg.get_item_rect_size("sidebar_window")[0] // 2
-    dpg.configure_item(group_tag, xoffset=value_column_width)
+      value_column_width = dpg.get_item_rect_size("sidebar_window")[0] // 2
+      dpg.configure_item(group_tag, xoffset=value_column_width)
 
-    value = self.data_manager.get_value_at(path, self.playback_manager.current_time_s)
-    if value is not None:
-      formatted_value = self.format_and_truncate(value, value_column_width, self.avg_char_width)
-      dpg.set_value(value_tag, formatted_value)
-    else:
-      dpg.set_value(value_tag, "N/A")
+      value = self.data_manager.get_value_at(path, self.playback_manager.current_time_s)
+      if value is not None:
+        formatted_value = self.format_and_truncate(value, value_column_width, self.avg_char_width)
+        dpg.set_value(value_tag, formatted_value)
+      else:
+        dpg.set_value(value_tag, "N/A")
 
   def _request_children_build(self, node: DataTreeNode, handler_tag=None):
-    if not node.children_ui_created and (node.name == "root" or (node.ui_tag is not None and dpg.get_value(node.ui_tag))):  # check root or node expanded
-      if handler_tag and dpg.does_item_exist(handler_tag):
-        dpg.delete_item(handler_tag)
-        self._item_handlers.discard(handler_tag)
-      parent_tag = "data_tree_container" if node.name == "root" else node.ui_tag
-      sorted_children = sorted(node.children.values(), key=self._natural_sort_key)
-
-      for i, child_node in enumerate(sorted_children):
-        if not child_node.ui_created:
-          before_tag: int | str = 0
-          for j in range(i + 1, len(sorted_children)):  # when incrementally building get "before_tag" for correct ordering
-            next_child = sorted_children[j]
-            if next_child.ui_created:
-              candidate_tag = f"group_{next_child.full_path}" if next_child.is_leaf else f"tree_{next_child.full_path}"
-              if dpg.does_item_exist(candidate_tag):
-                before_tag = candidate_tag
-                break
-          self.build_queue.append((child_node, parent_tag, before_tag))
-      node.children_ui_created = True
+    with self._ui_lock:
+      if not node.children_ui_created and (node.name == "root" or (node.ui_tag is not None and dpg.get_value(node.ui_tag))):  # check root or node expanded
+        parent_tag = "data_tree_container" if node.name == "root" else node.ui_tag
+        sorted_children = sorted(node.children.values(), key=self._natural_sort_key)
+
+        for i, child_node in enumerate(sorted_children):
+          if not child_node.ui_created:
+            before_tag: int | str = 0
+            for j in range(i + 1, len(sorted_children)):  # when incrementally building get "before_tag" for correct ordering
+              next_child = sorted_children[j]
+              if next_child.ui_created:
+                candidate_tag = f"group_{next_child.full_path}" if next_child.is_leaf else f"tree_{next_child.full_path}"
+                if dpg.does_item_exist(candidate_tag):
+                  before_tag = candidate_tag
+                  break
+            self.build_queue.append((child_node, parent_tag, before_tag))
+        node.children_ui_created = True
 
   def _should_show_path(self, path: str, search_term: str) -> bool:
     if 'DEPRECATED' in path and not os.environ.get('SHOW_DEPRECATED'):