allow verrryyy long search results

tools/jotpluggler/pluggle.py

@@ -66,6 +66,8 @@ class MainController:
     self.plot_layout_manager = PlotLayoutManager(self.data_manager, self.playback_manager, scale=self.scale)
     self.data_manager.add_observer(self.on_data_loaded)
     self.avg_char_width = None
+    self.visible_paths = set()
+    self.check_index = 0
 
   def _create_global_themes(self):
     with dpg.theme(tag="global_line_theme"):
@@ -94,9 +96,6 @@ class MainController:
     dpg.configure_item("timeline_slider", max_value=duration)
 
   def setup_ui(self):
-    with dpg.item_handler_registry(tag="tree_node_handler"):
-      dpg.add_item_toggled_open_handler(callback=self.data_tree_view.update_active_nodes_list)
-
     dpg.set_viewport_resize_callback(callback=self.on_viewport_resize)
 
     with dpg.window(tag="Primary Window"):
@@ -158,28 +157,49 @@ class MainController:
   def update_frame(self, font):
     with self.ui_lock:
       if self.avg_char_width is None:
-        self.avg_char_width = calculate_avg_char_width(font)  # must be calculated after first frame
+        self.avg_char_width = calculate_avg_char_width(font)
+      self.data_tree_view.update_frame()
 
       new_time = self.playback_manager.update_time(dpg.get_delta_time())
       if not dpg.is_item_active("timeline_slider"):
         dpg.set_value("timeline_slider", new_time)
 
       self._update_timeline_indicators(new_time)
+
       if not self.data_manager.loading and self.avg_char_width:
+        self._update_visible_set()
         self._update_data_values()
 
       dpg.set_value("fps_counter", f"{dpg.get_frame_rate():.1f} FPS")
 
+  def _update_visible_set(self): # for some reason, dpg has no way to easily check for visibility, and checking is slow...
+    all_paths = list(self.data_tree_view.created_leaf_paths)
+    if not all_paths:
+      self.visible_paths.clear()
+      return
+    chunk_size = min(50, len(all_paths)) # check up to 50 paths per frame
+    end_index = min(self.check_index + chunk_size, len(all_paths))
+    for i in range(self.check_index, end_index):
+      path = all_paths[i]
+      value_tag = f"value_{path}"
+      if dpg.does_item_exist(value_tag) and dpg.is_item_visible(value_tag):
+        self.visible_paths.add(path)
+      else:
+        self.visible_paths.discard(path)
+    self.check_index = end_index if end_index < len(all_paths) else 0
+
   def _update_data_values(self):
-    pool_width = dpg.get_item_rect_size("data_pool_window")[0]
+    value_column_width = dpg.get_item_rect_size("data_pool_window")[0] // 2
-    value_column_width = pool_width * 0.5
-    active_nodes = self.data_tree_view.active_leaf_nodes
 
-    for node in active_nodes:
+    for path in self.visible_paths.copy(): # avoid modification during iteration
-      path = node.full_path
       value_tag = f"value_{path}"
+      group_tag = f"group_{path}"
 
-      if dpg.does_item_exist(value_tag) and dpg.is_item_visible(value_tag):
+      if not dpg.does_item_exist(value_tag) or not dpg.does_item_exist(group_tag):
+        self.visible_paths.discard(path)
+        continue
+
+      dpg.configure_item(group_tag, xoffset=value_column_width)
       value = self.data_manager.get_value_at(path, self.playback_manager.current_time_s)
       if value is not None:
         formatted_value = format_and_truncate(value, value_column_width, self.avg_char_width)

tools/jotpluggler/views.py

@@ -2,6 +2,7 @@ import os
 import re
 import uuid
 import threading
+from collections import deque
 import dearpygui.dearpygui as dpg
 from abc import ABC, abstractmethod
 from openpilot.tools.jotpluggler.data import DataManager
@@ -183,36 +184,68 @@ class DataTreeNode:
     self.full_path = full_path
     self.children: dict[str, DataTreeNode] = {}
     self.is_leaf = False
+    self.child_count = 0
+    self.is_plottable_cached = None
+    self.ui_created = False
+    self.ui_tag: str | None = None
+
 
 class DataTreeView:
+  MAX_ITEMS_PER_FRAME = 50
+
   def __init__(self, data_manager: DataManager, ui_lock: threading.Lock):
     self.data_manager = data_manager
     self.ui_lock = ui_lock
     self.current_search = ""
     self.data_tree = DataTreeNode(name="root")
-    self.active_leaf_nodes: list[DataTreeNode] = []
+    self.ui_render_queue: deque[tuple[DataTreeNode, str, str, bool]] = deque() # (node, parent_tag, search_term, is_leaf)
-    self.data_manager.add_observer(self.on_data_loaded)
+    self.visible_expanded_nodes: set[str] = set()
-
+    self.created_leaf_paths: set[str] = set()
-  def on_data_loaded(self, data: dict):
+    self._all_paths_cache: list[str] = []
+    self.data_manager.add_observer(self._on_data_loaded)
+
+  def _on_data_loaded(self, data: dict):
+    if data.get('loading_complete'):
       with self.ui_lock:
-      self.populate_data_tree()
+        self._all_paths_cache = self.data_manager.get_all_paths()
+        self._populate_tree()
+
+  def update_frame(self):
+    items_processed = 0
+    while self.ui_render_queue and items_processed < self.MAX_ITEMS_PER_FRAME: # process up to MAX_ITEMS_PER_FRAME to maintain perforamnce
+      node, parent_tag, search_term, is_leaf = self.ui_render_queue.popleft()
+      if is_leaf:
+        self._create_leaf_ui(node, parent_tag)
+      else:
+        self._create_node_ui(node, parent_tag, search_term)
+      items_processed += 1
 
-  def populate_data_tree(self):
+  def search_data(self, search_term: str):
-    if not dpg.does_item_exist("data_tree_container"):
+    self.current_search = search_term
-      return
+    self._all_paths_cache = self.data_manager.get_all_paths()
+    self._populate_tree()
 
-    dpg.delete_item("data_tree_container", children_only=True)
+  def _populate_tree(self):
+    self._clear_ui()
     search_term = self.current_search.strip().lower()
+    self.data_tree = self._build_tree_structure(search_term)
+    for child in sorted(self.data_tree.children.values(), key=self._natural_sort_key): # queue top level nodes
+      self.ui_render_queue.append((child, "data_tree_container", search_term, child.is_leaf))
 
-    self.data_tree = DataTreeNode(name="root")
+  def _clear_ui(self):
-    all_paths = self.data_manager.get_all_paths()
+    dpg.delete_item("data_tree_container", children_only=True)
-
+    self.ui_render_queue.clear()
-    for path in sorted(all_paths):
+    self.visible_expanded_nodes.clear()
-      if not self._should_display_path(path, search_term):
+    self.created_leaf_paths.clear()
+
+  def _build_tree_structure(self, search_term: str) -> DataTreeNode:
+    root = DataTreeNode(name="root")
+    for path in sorted(self._all_paths_cache):
+      if not self._should_show_path(path, search_term):
         continue
 
       parts = path.split('/')
-      current_node = self.data_tree
+      current_node = root
       current_path_prefix = ""
 
       for part in parts:
@@ -220,76 +253,102 @@ class DataTreeView:
         if part not in current_node.children:
           current_node.children[part] = DataTreeNode(name=part, full_path=current_path_prefix)
         current_node = current_node.children[part]
-
       current_node.is_leaf = True
+    self._calculate_child_counts(root)
+    return root
 
-    self._create_ui_from_tree_recursive(self.data_tree, "data_tree_container", search_term)
+  def _calculate_child_counts(self, node: DataTreeNode):
-    self.update_active_nodes_list()
+    if node.is_leaf:
+      node.child_count = 0
+    else:
+      node.child_count = len(node.children)
+      for child in node.children.values():
+        self._calculate_child_counts(child)
 
-  def _should_display_path(self, path: str, search_term: str) -> bool:
+  def _create_node_ui(self, node: DataTreeNode, parent_tag: str, search_term: str):
-    if 'DEPRECATED' in path and not os.environ.get('SHOW_DEPRECATED'):
+    if node.is_leaf:
-      return False
+      self._create_leaf_ui(node, parent_tag)
-    return not search_term or search_term in path.lower()
+    else:
+      self._create_tree_node_ui(node, parent_tag, search_term)
 
-  def _natural_sort_key(self, node: DataTreeNode):
+  def _create_tree_node_ui(self, node: DataTreeNode, parent_tag: str, search_term: str):
-    node_type_key = node.is_leaf
+    node_tag = f"tree_{node.full_path}"
-    parts = [int(p) if p.isdigit() else p.lower() for p in re.split(r'(\d+)', node.name) if p]
+    node.ui_tag = node_tag
-    return (node_type_key, parts)
 
-  def _create_ui_from_tree_recursive(self, node: DataTreeNode, parent_tag: str, search_term: str):
+    label = f"{node.name} ({node.child_count} fields)"
-    sorted_children = sorted(node.children.values(), key=self._natural_sort_key)
+    should_open = (bool(search_term) and len(search_term) > 1 and any(search_term in path for path in self._get_descendant_paths(node)))
 
-    for child in sorted_children:
+    with dpg.tree_node(label=label, parent=parent_tag, tag=node_tag, default_open=should_open, open_on_arrow=True, open_on_double_click=True) as tree_node:
-      if child.is_leaf:
+      with dpg.item_handler_registry() as handler:
-        is_plottable = self.data_manager.is_plottable(child.full_path)
+        dpg.add_item_toggled_open_handler(callback=lambda s, d, u: self._on_node_expanded(node, search_term))
+      dpg.bind_item_handler_registry(tree_node, handler)
 
-        # Create draggable item
+    node.ui_created = True
-        with dpg.group(parent=parent_tag) as draggable_group:
-          with dpg.table(header_row=False, borders_innerV=False, borders_outerH=False, borders_outerV=False, policy=dpg.mvTable_SizingStretchProp):
-            dpg.add_table_column(init_width_or_weight=0.5)
-            dpg.add_table_column(init_width_or_weight=0.5)
-            with dpg.table_row():
-              dpg.add_text(child.name)
-              dpg.add_text("N/A", tag=f"value_{child.full_path}")
 
-        # Add drag payload if plottable
+    if should_open:
-        if is_plottable:
+      self.visible_expanded_nodes.add(node.full_path)
-          with dpg.drag_payload(parent=draggable_group, drag_data=child.full_path, payload_type="TIMESERIES_PAYLOAD"):
+      self._queue_children(node, node_tag, search_term)
-            dpg.add_text(f"Plot: {child.full_path}")
 
-      else:
+  def _create_leaf_ui(self, node: DataTreeNode, parent_tag: str):
-        node_tag = f"tree_{child.full_path}"
+    half_split_size = dpg.get_item_rect_size("data_pool_window")[0] // 2
-        label = child.name
+    with dpg.group(parent=parent_tag, horizontal=True, xoffset=half_split_size, tag=f"group_{node.full_path}") as draggable_group:
+      dpg.add_text(node.name)
+      dpg.add_text("N/A", tag=f"value_{node.full_path}")
+
+    if node.is_plottable_cached is None:
+      node.is_plottable_cached = self.data_manager.is_plottable(node.full_path)
+
+    if node.is_plottable_cached:
+      with dpg.drag_payload(parent=draggable_group, drag_data=node.full_path,
+                            payload_type="TIMESERIES_PAYLOAD"):
+        dpg.add_text(f"Plot: {node.full_path}")
+
+    node.ui_created = True
+    node.ui_tag = f"value_{node.full_path}"
+    self.created_leaf_paths.add(node.full_path)
+
+  def _queue_children(self, node: DataTreeNode, parent_tag: str, search_term: str):
+    for child in sorted(node.children.values(), key=self._natural_sort_key):
+      self.ui_render_queue.append((child, parent_tag, search_term, child.is_leaf))
+
+  def _on_node_expanded(self, node: DataTreeNode, search_term: str):
+    node_tag = f"tree_{node.full_path}"
+    if not dpg.does_item_exist(node_tag):
+      return
 
-        should_open = bool(search_term) and len(search_term) > 1 and any(search_term in path for path in self._get_all_descendant_paths(child))
+    is_expanded = dpg.get_value(node_tag)
 
-        with dpg.tree_node(label=label, parent=parent_tag, tag=node_tag, default_open=should_open):
+    if is_expanded:
-          dpg.bind_item_handler_registry(node_tag, "tree_node_handler")
+      if node.full_path not in self.visible_expanded_nodes:
-          self._create_ui_from_tree_recursive(child, node_tag, search_term)
+        self.visible_expanded_nodes.add(node.full_path)
+        self._queue_children(node, node_tag, search_term)
+    else:
+      self.visible_expanded_nodes.discard(node.full_path)
+      self._remove_children_from_queue(node.full_path)
+
+  def _remove_children_from_queue(self, collapsed_node_path: str):
+    new_queue = deque()
+    for node, parent_tag, search_term, is_leaf in self.ui_render_queue:
+      # Keep items that are not children of the collapsed node
+      if not node.full_path.startswith(collapsed_node_path + "/"):
+        new_queue.append((node, parent_tag, search_term, is_leaf))
+    self.ui_render_queue = new_queue
+
+  def _should_show_path(self, path: str, search_term: str) -> bool:
+    if 'DEPRECATED' in path and not os.environ.get('SHOW_DEPRECATED'):
+      return False
+    return not search_term or search_term in path.lower()
 
-  def _get_all_descendant_paths(self, node: DataTreeNode):
+  def _natural_sort_key(self, node: DataTreeNode):
+    node_type_key = node.is_leaf
+    parts = [int(p) if p.isdigit() else p.lower() for p in re.split(r'(\d+)', node.name) if p]
+    return (node_type_key, parts)
+
+  def _get_descendant_paths(self, node: DataTreeNode):
     for child_name, child_node in node.children.items():
       child_name_lower = child_name.lower()
       if child_node.is_leaf:
         yield child_name_lower
       else:
-        for path in self._get_all_descendant_paths(child_node):
+        for path in self._get_descendant_paths(child_node):
           yield f"{child_name_lower}/{path}"
-
-  def search_data(self, search_term: str):
-    self.current_search = search_term
-    self.populate_data_tree()
-
-  def update_active_nodes_list(self, sender=None, app_data=None, user_data=None):
-    self.active_leaf_nodes = self.get_active_leaf_nodes(self.data_tree)
-
-  def get_active_leaf_nodes(self, node: DataTreeNode):
-    active_leaves = []
-    for child in node.children.values():
-      if child.is_leaf:
-        active_leaves.append(child)
-      else:
-        node_tag = f"tree_{child.full_path}"
-        if dpg.does_item_exist(node_tag) and dpg.get_value(node_tag):
-          active_leaves.extend(self.get_active_leaf_nodes(child))
-    return active_leaves