diff --git a/tools/jotpluggler/pluggle.py b/tools/jotpluggler/pluggle.py index 614bb53e8e..d7a2cfba34 100755 --- a/tools/jotpluggler/pluggle.py +++ b/tools/jotpluggler/pluggle.py @@ -66,6 +66,8 @@ class MainController: self.plot_layout_manager = PlotLayoutManager(self.data_manager, self.playback_manager, scale=self.scale) self.data_manager.add_observer(self.on_data_loaded) self.avg_char_width = None + self.visible_paths = set() + self.check_index = 0 def _create_global_themes(self): with dpg.theme(tag="global_line_theme"): @@ -94,9 +96,6 @@ class MainController: dpg.configure_item("timeline_slider", max_value=duration) def setup_ui(self): - with dpg.item_handler_registry(tag="tree_node_handler"): - dpg.add_item_toggled_open_handler(callback=self.data_tree_view.update_active_nodes_list) - dpg.set_viewport_resize_callback(callback=self.on_viewport_resize) with dpg.window(tag="Primary Window"): @@ -158,32 +157,53 @@ class MainController: def update_frame(self, font): with self.ui_lock: if self.avg_char_width is None: - self.avg_char_width = calculate_avg_char_width(font) # must be calculated after first frame + self.avg_char_width = calculate_avg_char_width(font) + self.data_tree_view.update_frame() new_time = self.playback_manager.update_time(dpg.get_delta_time()) if not dpg.is_item_active("timeline_slider"): dpg.set_value("timeline_slider", new_time) self._update_timeline_indicators(new_time) + if not self.data_manager.loading and self.avg_char_width: + self._update_visible_set() self._update_data_values() dpg.set_value("fps_counter", f"{dpg.get_frame_rate():.1f} FPS") + def _update_visible_set(self): # for some reason, dpg has no way to easily check for visibility, and checking is slow... + all_paths = list(self.data_tree_view.created_leaf_paths) + if not all_paths: + self.visible_paths.clear() + return + chunk_size = min(50, len(all_paths)) # check up to 50 paths per frame + end_index = min(self.check_index + chunk_size, len(all_paths)) + for i in range(self.check_index, end_index): + path = all_paths[i] + value_tag = f"value_{path}" + if dpg.does_item_exist(value_tag) and dpg.is_item_visible(value_tag): + self.visible_paths.add(path) + else: + self.visible_paths.discard(path) + self.check_index = end_index if end_index < len(all_paths) else 0 + def _update_data_values(self): - pool_width = dpg.get_item_rect_size("data_pool_window")[0] - value_column_width = pool_width * 0.5 - active_nodes = self.data_tree_view.active_leaf_nodes + value_column_width = dpg.get_item_rect_size("data_pool_window")[0] // 2 - for node in active_nodes: - path = node.full_path + for path in self.visible_paths.copy(): # avoid modification during iteration value_tag = f"value_{path}" + group_tag = f"group_{path}" - if dpg.does_item_exist(value_tag) and dpg.is_item_visible(value_tag): - value = self.data_manager.get_value_at(path, self.playback_manager.current_time_s) - if value is not None: - formatted_value = format_and_truncate(value, value_column_width, self.avg_char_width) - dpg.set_value(value_tag, formatted_value) + if not dpg.does_item_exist(value_tag) or not dpg.does_item_exist(group_tag): + self.visible_paths.discard(path) + continue + + dpg.configure_item(group_tag, xoffset=value_column_width) + value = self.data_manager.get_value_at(path, self.playback_manager.current_time_s) + if value is not None: + formatted_value = format_and_truncate(value, value_column_width, self.avg_char_width) + dpg.set_value(value_tag, formatted_value) def _update_timeline_indicators(self, current_time_s: float): def update_node_recursive(node): diff --git a/tools/jotpluggler/views.py b/tools/jotpluggler/views.py index feb8abac69..bf089b4c6d 100644 --- a/tools/jotpluggler/views.py +++ b/tools/jotpluggler/views.py @@ -2,6 +2,7 @@ import os import re import uuid import threading +from collections import deque import dearpygui.dearpygui as dpg from abc import ABC, abstractmethod from openpilot.tools.jotpluggler.data import DataManager @@ -183,36 +184,68 @@ class DataTreeNode: self.full_path = full_path self.children: dict[str, DataTreeNode] = {} self.is_leaf = False + self.child_count = 0 + self.is_plottable_cached = None + self.ui_created = False + self.ui_tag: str | None = None + class DataTreeView: + MAX_ITEMS_PER_FRAME = 50 + def __init__(self, data_manager: DataManager, ui_lock: threading.Lock): self.data_manager = data_manager self.ui_lock = ui_lock self.current_search = "" self.data_tree = DataTreeNode(name="root") - self.active_leaf_nodes: list[DataTreeNode] = [] - self.data_manager.add_observer(self.on_data_loaded) - - def on_data_loaded(self, data: dict): - with self.ui_lock: - self.populate_data_tree() + self.ui_render_queue: deque[tuple[DataTreeNode, str, str, bool]] = deque() # (node, parent_tag, search_term, is_leaf) + self.visible_expanded_nodes: set[str] = set() + self.created_leaf_paths: set[str] = set() + self._all_paths_cache: list[str] = [] + self.data_manager.add_observer(self._on_data_loaded) + + def _on_data_loaded(self, data: dict): + if data.get('loading_complete'): + with self.ui_lock: + self._all_paths_cache = self.data_manager.get_all_paths() + self._populate_tree() + + def update_frame(self): + items_processed = 0 + while self.ui_render_queue and items_processed < self.MAX_ITEMS_PER_FRAME: # process up to MAX_ITEMS_PER_FRAME to maintain perforamnce + node, parent_tag, search_term, is_leaf = self.ui_render_queue.popleft() + if is_leaf: + self._create_leaf_ui(node, parent_tag) + else: + self._create_node_ui(node, parent_tag, search_term) + items_processed += 1 - def populate_data_tree(self): - if not dpg.does_item_exist("data_tree_container"): - return + def search_data(self, search_term: str): + self.current_search = search_term + self._all_paths_cache = self.data_manager.get_all_paths() + self._populate_tree() - dpg.delete_item("data_tree_container", children_only=True) + def _populate_tree(self): + self._clear_ui() search_term = self.current_search.strip().lower() + self.data_tree = self._build_tree_structure(search_term) + for child in sorted(self.data_tree.children.values(), key=self._natural_sort_key): # queue top level nodes + self.ui_render_queue.append((child, "data_tree_container", search_term, child.is_leaf)) - self.data_tree = DataTreeNode(name="root") - all_paths = self.data_manager.get_all_paths() - - for path in sorted(all_paths): - if not self._should_display_path(path, search_term): + def _clear_ui(self): + dpg.delete_item("data_tree_container", children_only=True) + self.ui_render_queue.clear() + self.visible_expanded_nodes.clear() + self.created_leaf_paths.clear() + + def _build_tree_structure(self, search_term: str) -> DataTreeNode: + root = DataTreeNode(name="root") + for path in sorted(self._all_paths_cache): + if not self._should_show_path(path, search_term): continue parts = path.split('/') - current_node = self.data_tree + current_node = root current_path_prefix = "" for part in parts: @@ -220,76 +253,102 @@ class DataTreeView: if part not in current_node.children: current_node.children[part] = DataTreeNode(name=part, full_path=current_path_prefix) current_node = current_node.children[part] - current_node.is_leaf = True + self._calculate_child_counts(root) + return root - self._create_ui_from_tree_recursive(self.data_tree, "data_tree_container", search_term) - self.update_active_nodes_list() + def _calculate_child_counts(self, node: DataTreeNode): + if node.is_leaf: + node.child_count = 0 + else: + node.child_count = len(node.children) + for child in node.children.values(): + self._calculate_child_counts(child) - def _should_display_path(self, path: str, search_term: str) -> bool: - if 'DEPRECATED' in path and not os.environ.get('SHOW_DEPRECATED'): - return False - return not search_term or search_term in path.lower() + def _create_node_ui(self, node: DataTreeNode, parent_tag: str, search_term: str): + if node.is_leaf: + self._create_leaf_ui(node, parent_tag) + else: + self._create_tree_node_ui(node, parent_tag, search_term) - def _natural_sort_key(self, node: DataTreeNode): - node_type_key = node.is_leaf - parts = [int(p) if p.isdigit() else p.lower() for p in re.split(r'(\d+)', node.name) if p] - return (node_type_key, parts) + def _create_tree_node_ui(self, node: DataTreeNode, parent_tag: str, search_term: str): + node_tag = f"tree_{node.full_path}" + node.ui_tag = node_tag - def _create_ui_from_tree_recursive(self, node: DataTreeNode, parent_tag: str, search_term: str): - sorted_children = sorted(node.children.values(), key=self._natural_sort_key) + label = f"{node.name} ({node.child_count} fields)" + should_open = (bool(search_term) and len(search_term) > 1 and any(search_term in path for path in self._get_descendant_paths(node))) - for child in sorted_children: - if child.is_leaf: - is_plottable = self.data_manager.is_plottable(child.full_path) + with dpg.tree_node(label=label, parent=parent_tag, tag=node_tag, default_open=should_open, open_on_arrow=True, open_on_double_click=True) as tree_node: + with dpg.item_handler_registry() as handler: + dpg.add_item_toggled_open_handler(callback=lambda s, d, u: self._on_node_expanded(node, search_term)) + dpg.bind_item_handler_registry(tree_node, handler) - # Create draggable item - with dpg.group(parent=parent_tag) as draggable_group: - with dpg.table(header_row=False, borders_innerV=False, borders_outerH=False, borders_outerV=False, policy=dpg.mvTable_SizingStretchProp): - dpg.add_table_column(init_width_or_weight=0.5) - dpg.add_table_column(init_width_or_weight=0.5) - with dpg.table_row(): - dpg.add_text(child.name) - dpg.add_text("N/A", tag=f"value_{child.full_path}") + node.ui_created = True - # Add drag payload if plottable - if is_plottable: - with dpg.drag_payload(parent=draggable_group, drag_data=child.full_path, payload_type="TIMESERIES_PAYLOAD"): - dpg.add_text(f"Plot: {child.full_path}") + if should_open: + self.visible_expanded_nodes.add(node.full_path) + self._queue_children(node, node_tag, search_term) - else: - node_tag = f"tree_{child.full_path}" - label = child.name + def _create_leaf_ui(self, node: DataTreeNode, parent_tag: str): + half_split_size = dpg.get_item_rect_size("data_pool_window")[0] // 2 + with dpg.group(parent=parent_tag, horizontal=True, xoffset=half_split_size, tag=f"group_{node.full_path}") as draggable_group: + dpg.add_text(node.name) + dpg.add_text("N/A", tag=f"value_{node.full_path}") + + if node.is_plottable_cached is None: + node.is_plottable_cached = self.data_manager.is_plottable(node.full_path) + + if node.is_plottable_cached: + with dpg.drag_payload(parent=draggable_group, drag_data=node.full_path, + payload_type="TIMESERIES_PAYLOAD"): + dpg.add_text(f"Plot: {node.full_path}") - should_open = bool(search_term) and len(search_term) > 1 and any(search_term in path for path in self._get_all_descendant_paths(child)) + node.ui_created = True + node.ui_tag = f"value_{node.full_path}" + self.created_leaf_paths.add(node.full_path) + + def _queue_children(self, node: DataTreeNode, parent_tag: str, search_term: str): + for child in sorted(node.children.values(), key=self._natural_sort_key): + self.ui_render_queue.append((child, parent_tag, search_term, child.is_leaf)) + + def _on_node_expanded(self, node: DataTreeNode, search_term: str): + node_tag = f"tree_{node.full_path}" + if not dpg.does_item_exist(node_tag): + return + + is_expanded = dpg.get_value(node_tag) + + if is_expanded: + if node.full_path not in self.visible_expanded_nodes: + self.visible_expanded_nodes.add(node.full_path) + self._queue_children(node, node_tag, search_term) + else: + self.visible_expanded_nodes.discard(node.full_path) + self._remove_children_from_queue(node.full_path) + + def _remove_children_from_queue(self, collapsed_node_path: str): + new_queue = deque() + for node, parent_tag, search_term, is_leaf in self.ui_render_queue: + # Keep items that are not children of the collapsed node + if not node.full_path.startswith(collapsed_node_path + "/"): + new_queue.append((node, parent_tag, search_term, is_leaf)) + self.ui_render_queue = new_queue + + def _should_show_path(self, path: str, search_term: str) -> bool: + if 'DEPRECATED' in path and not os.environ.get('SHOW_DEPRECATED'): + return False + return not search_term or search_term in path.lower() - with dpg.tree_node(label=label, parent=parent_tag, tag=node_tag, default_open=should_open): - dpg.bind_item_handler_registry(node_tag, "tree_node_handler") - self._create_ui_from_tree_recursive(child, node_tag, search_term) + def _natural_sort_key(self, node: DataTreeNode): + node_type_key = node.is_leaf + parts = [int(p) if p.isdigit() else p.lower() for p in re.split(r'(\d+)', node.name) if p] + return (node_type_key, parts) - def _get_all_descendant_paths(self, node: DataTreeNode): + def _get_descendant_paths(self, node: DataTreeNode): for child_name, child_node in node.children.items(): child_name_lower = child_name.lower() if child_node.is_leaf: yield child_name_lower else: - for path in self._get_all_descendant_paths(child_node): + for path in self._get_descendant_paths(child_node): yield f"{child_name_lower}/{path}" - - def search_data(self, search_term: str): - self.current_search = search_term - self.populate_data_tree() - - def update_active_nodes_list(self, sender=None, app_data=None, user_data=None): - self.active_leaf_nodes = self.get_active_leaf_nodes(self.data_tree) - - def get_active_leaf_nodes(self, node: DataTreeNode): - active_leaves = [] - for child in node.children.values(): - if child.is_leaf: - active_leaves.append(child) - else: - node_tag = f"tree_{child.full_path}" - if dpg.does_item_exist(node_tag) and dpg.get_value(node_tag): - active_leaves.extend(self.get_active_leaf_nodes(child)) - return active_leaves