simplify and speed up timeseries

tools/jotpluggler/data.py

@@ -200,7 +200,7 @@ class DataManager:
           values.append(segment[msg_type][field])
 
       if not times:
-        return None
+        return [], []
 
       combined_times = np.concatenate(times) - self._start_time
       if len(values) > 1 and any(arr.dtype != values[0].dtype for arr in values):

tools/jotpluggler/layout.py

@@ -18,10 +18,6 @@ class LayoutNode(ABC):
   def destroy_ui(self):
     pass
 
-  @abstractmethod
-  def preserve_data(self):
-    pass
-
 
 class LeafNode(LayoutNode):
   """Leaf node that contains a single ViewPanel with controls"""
@@ -32,9 +28,6 @@ class LeafNode(LayoutNode):
     self.layout_manager = layout_manager
     self.scale = scale
 
-  def preserve_data(self):
-    self.panel.preserve_data()
-
   def create_ui(self, parent_tag: str, width: int = -1, height: int = -1):
     """Create UI container with controls and panel"""
     self.tag = f"leaf_{self.node_id}"
@@ -95,10 +88,6 @@ class SplitterNode(LayoutNode):
     self.child_proportions = [1.0 / len(self.children) for _ in self.children] if self.children else []
     self.child_container_tags: list[str] = []  # Track container tags for resizing
 
-  def preserve_data(self):
-    for child in self.children:
-      child.preserve_data()
-
   def add_child(self, child: LayoutNode, index: int = None):
     if index is None:
       self.children.append(child)
@@ -219,14 +208,12 @@ class PlotLayoutManager:
     parent_node, child_index = self._find_parent_and_index(node)
 
     if parent_node is None:  # root node - create new splitter as root
-      node.preserve_data()
       self.root_node = SplitterNode([node, new_leaf], orientation)
       self._update_ui_for_node(self.root_node, self.container_tag)
     elif isinstance(parent_node, SplitterNode) and parent_node.orientation == orientation:  # same orientation - add to existing splitter
       parent_node.add_child(new_leaf, child_index + 1)
       self._update_ui_for_node(parent_node)
     else:  # different orientation - replace node with new splitter
-      node.preserve_data()
       new_splitter = SplitterNode([node, new_leaf], orientation)
       self._replace_child_in_parent(parent_node, node, new_splitter)
 
@@ -244,7 +231,6 @@ class PlotLayoutManager:
         grandparent_node, parent_index = self._find_parent_and_index(parent_node)
 
         if grandparent_node is None:  # promote remaining child to root
-          remaining_child.preserve_data()
           parent_node.children.remove(remaining_child)
           self.root_node = remaining_child
           parent_node.destroy_ui()
@@ -255,8 +241,6 @@ class PlotLayoutManager:
         self._update_ui_for_node(parent_node)
 
   def _replace_child_in_parent(self, parent_node: SplitterNode, old_child: LayoutNode, new_child: LayoutNode):
-    old_child.preserve_data()  # save data and for when recreating ui for the node
-
     child_index = parent_node.children.index(old_child)
     child_container_tag = f"child_container_{parent_node.node_id}_{child_index}"
 
@@ -272,8 +256,6 @@ class PlotLayoutManager:
       new_child.create_ui(child_container_tag, container_width, container_height)
 
   def _update_ui_for_node(self, node: LayoutNode, container_tag: str = None):
-    node.preserve_data()
-
     if container_tag:  #  update node in a specific container (usually root)
       dpg.delete_item(container_tag, children_only=True)
       container_width, container_height = dpg.get_item_rect_size(container_tag)

tools/jotpluggler/pluggle.py

@@ -166,7 +166,7 @@ class MainController:
 
       self._update_timeline_indicators(new_time)
 
-      if not self.data_manager.loading and self.avg_char_width:
+      if self.avg_char_width:
         self._update_visible_set()
         self._update_data_values()
 

tools/jotpluggler/views.py

@@ -2,6 +2,7 @@ import os
 import re
 import uuid
 import threading
+import numpy as np
 from collections import deque
 import dearpygui.dearpygui as dpg
 from abc import ABC, abstractmethod
@@ -27,10 +28,6 @@ class ViewPanel(ABC):
   def get_panel_type(self) -> str:
     pass
 
-  @abstractmethod
-  def preserve_data(self):
-    pass
-
 
 class TimeSeriesPanel(ViewPanel):
   def __init__(self, data_manager: DataManager, playback_manager, panel_id: str | None = None):
@@ -38,24 +35,14 @@ class TimeSeriesPanel(ViewPanel):
     self.data_manager = data_manager
     self.playback_manager = playback_manager
     self.title = "Time Series Plot"
-    self.plotted_series: set[str] = set()
     self.plot_tag: str | None = None
     self.x_axis_tag: str | None = None
     self.y_axis_tag: str | None = None
     self.timeline_indicator_tag: str | None = None
     self._ui_created = False
-    self._preserved_series_data: list[tuple[str, tuple]] = []  # TODO: the way we do this right now doesn't make much sense
+    self._series_data: dict[str, tuple] = {}
-    self._series_legend_tags: dict[str, str] = {}  # Maps series_path to legend tag
     self.data_manager.add_observer(self.on_data_loaded)
 
-  def preserve_data(self):
-    self._preserved_series_data = []
-    if self.plotted_series and self._ui_created:
-      for series_path in self.plotted_series:
-        time_value_data = self.data_manager.get_timeseries(series_path)
-        if time_value_data:
-          self._preserved_series_data.append((series_path, time_value_data))
-
   def create_ui(self, parent_tag: str):
     self.plot_tag = f"plot_{self.panel_id}"
     self.x_axis_tag = f"{self.plot_tag}_x_axis"
@@ -64,18 +51,13 @@ class TimeSeriesPanel(ViewPanel):
 
     with dpg.plot(height=-1, width=-1, tag=self.plot_tag, parent=parent_tag, drop_callback=self._on_series_drop, payload_type="TIMESERIES_PAYLOAD"):
       dpg.add_plot_legend()
-      dpg.add_plot_axis(dpg.mvXAxis, label="", tag=self.x_axis_tag)
+      dpg.add_plot_axis(dpg.mvXAxis, no_label=True, tag=self.x_axis_tag)
-      dpg.add_plot_axis(dpg.mvYAxis, label="", tag=self.y_axis_tag)
+      dpg.add_plot_axis(dpg.mvYAxis, no_label=True, tag=self.y_axis_tag)
-
       timeline_series_tag = dpg.add_inf_line_series(x=[0], label="Timeline", parent=self.y_axis_tag, tag=self.timeline_indicator_tag)
       dpg.bind_item_theme(timeline_series_tag, "global_timeline_theme")
 
-    # Restore series from preserved data
+    for series_path in list(self._series_data.keys()):
-    if self._preserved_series_data:
+      self.add_series(series_path)
-      self.plotted_series.clear()
-      for series_path, (rel_time_array, value_array) in self._preserved_series_data:
-        self._add_series_with_data(series_path, rel_time_array, value_array)
-      self._preserved_series_data = []
 
     self._ui_created = True
 
@@ -83,99 +65,68 @@ class TimeSeriesPanel(ViewPanel):
     if not self._ui_created or not dpg.does_item_exist(self.timeline_indicator_tag):
       return
 
-    dpg.set_value(self.timeline_indicator_tag, [[current_time_s], [0]])  # vertical line position
+    dpg.set_value(self.timeline_indicator_tag, [[current_time_s], [0]])
 
-    if self.plotted_series:  # update legend labels with current values
+    for series_path, (rel_time_array, value_array) in self._series_data.items():
-      for series_path in self.plotted_series:
+      position = np.searchsorted(rel_time_array, current_time_s, side='right') - 1
-        value = self.data_manager.get_value_at(series_path, current_time_s)
+      value = None
 
-        if value is not None:
+      if position >= 0 and (current_time_s - rel_time_array[position]) <= 1.0:
-          if isinstance(value, (int, float)):
+        value = value_array[position]
-            if isinstance(value, float):
-              formatted_value = f"{value:.4f}" if abs(value) < 1000 else f"{value:.3e}"
-            else:
-              formatted_value = str(value)
-          else:
-            formatted_value = str(value)
 
-          series_tag = f"series_{self.panel_id}_{series_path.replace('/', '_')}"
+      if value is not None:
-          legend_label = f"{series_path}: {formatted_value}"
+        if isinstance(value, float):
+          formatted_value = f"{value:.4f}" if abs(value) < 1000 else f"{value:.3e}"
+        else:
+          formatted_value = str(value)
 
-          if dpg.does_item_exist(series_tag):
+        series_tag = f"series_{self.panel_id}_{series_path}"
-            dpg.configure_item(series_tag, label=legend_label)
+        legend_label = f"{series_path}: {formatted_value}"
 
-  def _add_series_with_data(self, series_path: str, rel_time_array, value_array) -> bool:
+        if dpg.does_item_exist(series_tag):
-    if series_path in self.plotted_series:
+          dpg.configure_item(series_tag, label=legend_label)
-      return False
 
-    series_tag = f"series_{self.panel_id}_{series_path.replace('/', '_')}"
+  def add_series(self, series_path: str, update: bool = False) -> bool:
-    line_series_tag = dpg.add_line_series(x=rel_time_array.tolist(), y=value_array.tolist(), label=series_path, parent=self.y_axis_tag, tag=series_tag)
+    if update or series_path not in self._series_data:
+      self._series_data[series_path] = self.data_manager.get_timeseries(series_path)
 
-    dpg.bind_item_theme(line_series_tag, "global_line_theme")
+    rel_time_array, value_array = self._series_data[series_path]
+    series_tag = f"series_{self.panel_id}_{series_path}"
 
-    self.plotted_series.add(series_path)
+    if dpg.does_item_exist(series_tag):
-    dpg.fit_axis_data(self.x_axis_tag)
+      dpg.set_value(series_tag, [rel_time_array, value_array])
-    dpg.fit_axis_data(self.y_axis_tag)
+    else:
+      line_series_tag = dpg.add_line_series(x=rel_time_array, y=value_array, label=series_path, parent=self.y_axis_tag, tag=series_tag)
+      dpg.bind_item_theme(line_series_tag, "global_line_theme")
+      dpg.fit_axis_data(self.x_axis_tag)
+      dpg.fit_axis_data(self.y_axis_tag)
     return True
 
   def destroy_ui(self):
     if self.plot_tag and dpg.does_item_exist(self.plot_tag):
       dpg.delete_item(self.plot_tag)
-
-    self._series_legend_tags.clear()
     self._ui_created = False
 
   def get_panel_type(self) -> str:
     return "timeseries"
 
-  def add_series(self, series_path: str) -> bool:
-    if series_path in self.plotted_series:
-      return False
-
-    time_value_data = self.data_manager.get_timeseries(series_path)
-    if time_value_data is None:
-      return False
-
-    rel_time_array, value_array = time_value_data
-    return self._add_series_with_data(series_path, rel_time_array, value_array)
-
   def clear_all_series(self):
-    for series_path in self.plotted_series.copy():
+    for series_path in list(self._series_data.keys()):
       self.remove_series(series_path)
 
   def remove_series(self, series_path: str):
-    if series_path in self.plotted_series:
+    if series_path in self._series_data:
-      series_tag = f"series_{self.panel_id}_{series_path.replace('/', '_')}"
+      series_tag = f"series_{self.panel_id}_{series_path}"
       if dpg.does_item_exist(series_tag):
         dpg.delete_item(series_tag)
-      self.plotted_series.remove(series_path)
+      del self._series_data[series_path]
-      if series_path in self._series_legend_tags:
-        del self._series_legend_tags[series_path]
 
   def on_data_loaded(self, data: dict):
-    for series_path in self.plotted_series.copy():
+    for series_path in list(self._series_data.keys()):
-      self._update_series_data(series_path)
+      self.add_series(series_path, update=True)
-
-  def _update_series_data(self, series_path: str) -> bool:
-    time_value_data = self.data_manager.get_timeseries(series_path)
-    if time_value_data is None:
-      return False
-
-    rel_time_array, value_array = time_value_data
-    series_tag = f"series_{self.panel_id}_{series_path.replace('/', '_')}"
-
-    if dpg.does_item_exist(series_tag):
-      dpg.set_value(series_tag, [rel_time_array.tolist(), value_array.tolist()])
-      dpg.fit_axis_data(self.x_axis_tag)
-      dpg.fit_axis_data(self.y_axis_tag)
-      return True
-    else:
-      self.plotted_series.discard(series_path)
-      return False
 
   def _on_series_drop(self, sender, app_data, user_data):
-    series_path = app_data
+    self.add_series(app_data)
-    self.add_series(series_path)
 
 
 class DataTreeNode: