From 2e41d959ac8610f2a91d28627c0f1357b1089453 Mon Sep 17 00:00:00 2001 From: Dean Lee Date: Wed, 4 Jun 2025 14:12:36 +0800 Subject: [PATCH] ui: add main UI entry point (#35422) * add new main UI entry point * cleanup * mv to selfdrive/ui * fix imports * handle_mouse_click * use ui_state * remove ui_state from gui_app * setup callbacks * handle clicks * put layouts in a dict * update state in render * rebase master * implement settings sidebar * rename files --- selfdrive/ui/layouts/__init__.py | 0 selfdrive/ui/layouts/home.py | 17 ++ selfdrive/ui/layouts/main.py | 92 +++++++++ selfdrive/ui/layouts/settings/settings.py | 174 +++++++++++++++++ selfdrive/ui/layouts/sidebar.py | 207 +++++++++++++++++++++ selfdrive/ui/onroad/augmented_road_view.py | 2 +- selfdrive/ui/ui.py | 20 ++ 7 files changed, 511 insertions(+), 1 deletion(-) create mode 100644 selfdrive/ui/layouts/__init__.py create mode 100644 selfdrive/ui/layouts/home.py create mode 100644 selfdrive/ui/layouts/main.py create mode 100644 selfdrive/ui/layouts/settings/settings.py create mode 100644 selfdrive/ui/layouts/sidebar.py create mode 100755 selfdrive/ui/ui.py diff --git a/selfdrive/ui/layouts/__init__.py b/selfdrive/ui/layouts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/selfdrive/ui/layouts/home.py b/selfdrive/ui/layouts/home.py new file mode 100644 index 0000000000..56df253b52 --- /dev/null +++ b/selfdrive/ui/layouts/home.py @@ -0,0 +1,17 @@ +import pyray as rl +from openpilot.system.ui.lib.label import gui_text_box + + +class HomeLayout: + def __init__(self): + pass + + def render(self, rect: rl.Rectangle): + gui_text_box( + rect, + "Demo Home Layout", + font_size=170, + color=rl.WHITE, + alignment=rl.GuiTextAlignment.TEXT_ALIGN_CENTER, + alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE, + ) diff --git a/selfdrive/ui/layouts/main.py b/selfdrive/ui/layouts/main.py new file mode 100644 index 0000000000..2096480f22 --- /dev/null +++ b/selfdrive/ui/layouts/main.py @@ -0,0 +1,92 @@ +import pyray as rl +from enum import IntEnum +from openpilot.selfdrive.ui.layouts.sidebar import Sidebar, SIDEBAR_WIDTH +from openpilot.selfdrive.ui.layouts.home import HomeLayout +from openpilot.selfdrive.ui.layouts.settings.settings import SettingsLayout +from openpilot.selfdrive.ui.ui_state import ui_state +from openpilot.selfdrive.ui.onroad.augmented_road_view import AugmentedRoadView + + +class MainState(IntEnum): + HOME = 0 + SETTINGS = 1 + ONROAD = 2 + + +class MainLayout: + def __init__(self): + self._sidebar = Sidebar() + self._sidebar_visible = True + self._current_mode = MainState.HOME + self._prev_onroad = False + self._window_rect = None + self._current_callback: callable | None = None + + # Initialize layouts + self._layouts = {MainState.HOME: HomeLayout(), MainState.SETTINGS: SettingsLayout(), MainState.ONROAD: AugmentedRoadView()} + + self._sidebar_rect = rl.Rectangle(0, 0, 0, 0) + self._content_rect = rl.Rectangle(0, 0, 0, 0) + + # Set callbacks + self._setup_callbacks() + + def render(self, rect): + self._current_callback = None + + self._update_layout_rects(rect) + self._render_main_content() + self._handle_input() + + if self._current_callback: + self._current_callback() + + def _setup_callbacks(self): + self._sidebar.set_callbacks( + on_settings=lambda: setattr(self, '_current_callback', self._on_settings_clicked), + on_flag=lambda: setattr(self, '_current_callback', self._on_flag_clicked), + ) + self._layouts[MainState.SETTINGS].set_callbacks( + on_close=lambda: setattr(self, '_current_callback', self._on_settings_closed) + ) + + def _update_layout_rects(self, rect): + self._window_rect = rect + self._sidebar_rect = rl.Rectangle(rect.x, rect.y, SIDEBAR_WIDTH, rect.height) + + x_offset = SIDEBAR_WIDTH if self._sidebar_visible else 0 + self._content_rect = rl.Rectangle(rect.y + x_offset, rect.y, rect.width - x_offset, rect.height) + + def _on_settings_clicked(self): + self._current_mode = MainState.SETTINGS + self._sidebar_visible = False + + def _on_settings_closed(self): + self._current_mode = MainState.HOME if not ui_state.started else MainState.ONROAD + self._sidebar_visible = True + + def _on_flag_clicked(self): + pass + + def _render_main_content(self): + # Render sidebar + if self._sidebar_visible: + self._sidebar.render(self._sidebar_rect) + + if ui_state.started != self._prev_onroad: + self._prev_onroad = ui_state.started + if ui_state.started: + self._current_mode = MainState.ONROAD + else: + self._current_mode = MainState.HOME + + content_rect = self._content_rect if self._sidebar_visible else self._window_rect + self._layouts[self._current_mode].render(content_rect) + + def _handle_input(self): + if self._current_mode != MainState.ONROAD or not rl.is_mouse_button_pressed(rl.MouseButton.MOUSE_BUTTON_LEFT): + return + + mouse_pos = rl.get_mouse_position() + if rl.check_collision_point_rec(mouse_pos, self._content_rect): + self._sidebar_visible = not self._sidebar_visible diff --git a/selfdrive/ui/layouts/settings/settings.py b/selfdrive/ui/layouts/settings/settings.py new file mode 100644 index 0000000000..6a9a50b855 --- /dev/null +++ b/selfdrive/ui/layouts/settings/settings.py @@ -0,0 +1,174 @@ +import pyray as rl +from dataclasses import dataclass +from enum import IntEnum +from collections.abc import Callable +from openpilot.common.params import Params +from openpilot.system.ui.lib.application import gui_app, FontWeight +from openpilot.system.ui.lib.label import gui_text_box + +# Import individual panels + +SETTINGS_CLOSE_TEXT = "X" +# Constants +SIDEBAR_WIDTH = 500 +CLOSE_BTN_SIZE = 200 +NAV_BTN_HEIGHT = 80 +PANEL_MARGIN = 50 +SCROLL_SPEED = 30 + +# Colors +SIDEBAR_COLOR = rl.BLACK +PANEL_COLOR = rl.Color(41, 41, 41, 255) +CLOSE_BTN_COLOR = rl.Color(41, 41, 41, 255) +CLOSE_BTN_PRESSED = rl.Color(59, 59, 59, 255) +TEXT_NORMAL = rl.Color(128, 128, 128, 255) +TEXT_SELECTED = rl.Color(255, 255, 255, 255) +TEXT_PRESSED = rl.Color(173, 173, 173, 255) + + +class PanelType(IntEnum): + DEVICE = 0 + NETWORK = 1 + TOGGLES = 2 + SOFTWARE = 3 + FIREHOSE = 4 + DEVELOPER = 5 + + +@dataclass +class PanelInfo: + name: str + instance: object + button_rect: rl.Rectangle + + +class SettingsLayout: + def __init__(self): + self._params = Params() + self._current_panel = PanelType.DEVICE + self._close_btn_pressed = False + self._scroll_offset = 0.0 + self._max_scroll = 0.0 + + # Panel configuration + self._panels = { + PanelType.DEVICE: PanelInfo("Device", None, rl.Rectangle(0, 0, 0, 0)), + PanelType.TOGGLES: PanelInfo("Toggles", None, rl.Rectangle(0, 0, 0, 0)), + PanelType.SOFTWARE: PanelInfo("Software", None, rl.Rectangle(0, 0, 0, 0)), + PanelType.FIREHOSE: PanelInfo("Firehose", None, rl.Rectangle(0, 0, 0, 0)), + PanelType.NETWORK: PanelInfo("Network", None, rl.Rectangle(0, 0, 0, 0)), + PanelType.DEVELOPER: PanelInfo("Developer", None, rl.Rectangle(0, 0, 0, 0)), + } + + self._font_medium = gui_app.font(FontWeight.MEDIUM) + self._font_bold = gui_app.font(FontWeight.SEMI_BOLD) + + # Callbacks + self._close_callback: Callable | None = None + + def set_callbacks(self, on_close: Callable): + self._close_callback = on_close + + def render(self, rect: rl.Rectangle): + # Calculate layout + sidebar_rect = rl.Rectangle(rect.x, rect.y, SIDEBAR_WIDTH, rect.height) + panel_rect = rl.Rectangle(rect.x + SIDEBAR_WIDTH, rect.y, rect.width - SIDEBAR_WIDTH, rect.height) + + # Draw components + self._draw_sidebar(sidebar_rect) + self._draw_current_panel(panel_rect) + + if rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT): + self.handle_mouse_release(rl.get_mouse_position()) + + def _draw_sidebar(self, rect: rl.Rectangle): + rl.draw_rectangle_rec(rect, SIDEBAR_COLOR) + + # Close button + close_btn_rect = rl.Rectangle( + rect.x + (rect.width - CLOSE_BTN_SIZE) / 2, rect.y + 45, CLOSE_BTN_SIZE, CLOSE_BTN_SIZE + ) + + close_color = CLOSE_BTN_PRESSED if self._close_btn_pressed else CLOSE_BTN_COLOR + rl.draw_rectangle_rounded(close_btn_rect, 0.5, 20, close_color) + close_text_size = rl.measure_text_ex(self._font_bold, SETTINGS_CLOSE_TEXT, 140, 0) + close_text_pos = rl.Vector2( + close_btn_rect.x + (close_btn_rect.width - close_text_size.x) / 2, + close_btn_rect.y + (close_btn_rect.height - close_text_size.y) / 2 - 20, + ) + rl.draw_text_ex(self._font_bold, SETTINGS_CLOSE_TEXT, close_text_pos, 140, 0, TEXT_SELECTED) + + # Store close button rect for click detection + self._close_btn_rect = close_btn_rect + + # Navigation buttons + nav_start_y = rect.y + 300 + button_spacing = 20 + + i = 0 + for panel_type, panel_info in self._panels.items(): + button_rect = rl.Rectangle( + rect.x + 50, + nav_start_y + i * (NAV_BTN_HEIGHT + button_spacing), + rect.width - 150, # Right-aligned with margin + NAV_BTN_HEIGHT, + ) + + # Button styling + is_selected = panel_type == self._current_panel + text_color = TEXT_SELECTED if is_selected else TEXT_NORMAL + + # Draw button text (right-aligned) + text_size = rl.measure_text_ex(self._font_medium, panel_info.name, 65, 0) + text_pos = rl.Vector2( + button_rect.x + button_rect.width - text_size.x, button_rect.y + (button_rect.height - text_size.y) / 2 + ) + rl.draw_text_ex(self._font_medium, panel_info.name, text_pos, 65, 0, text_color) + + # Store button rect for click detection + panel_info.button_rect = button_rect + i += 1 + + def _draw_current_panel(self, rect: rl.Rectangle): + content_rect = rl.Rectangle(rect.x + PANEL_MARGIN, rect.y + 25, rect.width - (PANEL_MARGIN * 2), rect.height - 50) + rl.draw_rectangle_rounded(content_rect, 0.03, 30, PANEL_COLOR) + gui_text_box( + content_rect, + f"Demo {self._panels[self._current_panel].name} Panel", + font_size=170, + color=rl.WHITE, + alignment=rl.GuiTextAlignment.TEXT_ALIGN_CENTER, + alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE, + ) + + def handle_mouse_release(self, mouse_pos: rl.Vector2) -> bool: + # Check close button + if rl.check_collision_point_rec(mouse_pos, self._close_btn_rect): + self._close_btn_pressed = True + if self._close_callback: + self._close_callback() + return True + + # Check navigation buttons + for panel_type, panel_info in self._panels.items(): + if rl.check_collision_point_rec(mouse_pos, panel_info.button_rect): + self._switch_to_panel(panel_type) + return True + + return False + + def _switch_to_panel(self, panel_type: PanelType): + if panel_type != self._current_panel: + self._current_panel = panel_type + self._scroll_offset = 0.0 # Reset scroll when switching panels + self._transition_progress = 0.0 + self._transitioning = True + + def set_current_panel(self, index: int, param: str = ""): + panel_types = list(self._panels.keys()) + if 0 <= index < len(panel_types): + self._switch_to_panel(panel_types[index]) + + def close_settings(self): + if self._close_callback: + self._close_callback() diff --git a/selfdrive/ui/layouts/sidebar.py b/selfdrive/ui/layouts/sidebar.py new file mode 100644 index 0000000000..eeb073e67b --- /dev/null +++ b/selfdrive/ui/layouts/sidebar.py @@ -0,0 +1,207 @@ +import pyray as rl +import time +from dataclasses import dataclass +from collections.abc import Callable +from cereal import log +from openpilot.selfdrive.ui.ui_state import ui_state +from openpilot.system.ui.lib.application import gui_app, FontWeight + +SIDEBAR_WIDTH = 300 +METRIC_HEIGHT = 126 +METRIC_WIDTH = 240 +METRIC_MARGIN = 30 + +SETTINGS_BTN = rl.Rectangle(50, 35, 200, 117) +HOME_BTN = rl.Rectangle(60, 860, 180, 180) + +ThermalStatus = log.DeviceState.ThermalStatus +NetworkType = log.DeviceState.NetworkType + +# Color scheme +class Colors: + SIDEBAR_BG = rl.Color(57, 57, 57, 255) + WHITE = rl.Color(255, 255, 255, 255) + WHITE_DIM = rl.Color(255, 255, 255, 85) + GRAY = rl.Color(84, 84, 84, 255) + + # Status colors + GOOD = rl.Color(255, 255, 255, 255) + WARNING = rl.Color(218, 202, 37, 255) + DANGER = rl.Color(201, 34, 49, 255) + + # UI elements + METRIC_BORDER = rl.Color(255, 255, 255, 85) + BUTTON_NORMAL = rl.Color(255, 255, 255, 255) + BUTTON_PRESSED = rl.Color(255, 255, 255, 166) + +NETWORK_TYPES = { + NetworkType.none: "Offline", + NetworkType.wifi: "WiFi", + NetworkType.cell2G: "2G", + NetworkType.cell3G: "3G", + NetworkType.cell4G: "LTE", + NetworkType.cell5G: "5G", + NetworkType.ethernet: "Ethernet", +} + + +@dataclass(slots=True) +class MetricData: + label: str + value: str + color: rl.Color + + def update(self, label: str, value: str, color: rl.Color): + self.label = label + self.value = value + self.color = color + +class Sidebar: + def __init__(self): + self._net_type = NETWORK_TYPES.get(NetworkType.none) + self._net_strength = 0 + + self._temp_status = MetricData("TEMP", "GOOD", Colors.GOOD) + self._panda_status = MetricData("VEHICLE", "ONLINE", Colors.GOOD) + self._connect_status = MetricData("CONNECT", "OFFLINE", Colors.WARNING) + + self._home_img = gui_app.texture("images/button_home.png", HOME_BTN.width, HOME_BTN.height) + self._flag_img = gui_app.texture("images/button_flag.png", HOME_BTN.width, HOME_BTN.height) + self._settings_img = gui_app.texture("images/button_settings.png", SETTINGS_BTN.width, SETTINGS_BTN.height) + self._font_regular = gui_app.font(FontWeight.NORMAL) + self._font_bold = gui_app.font(FontWeight.SEMI_BOLD) + + # Callbacks + self._on_settings_click: Callable | None = None + self._on_flag_click: Callable | None = None + + def set_callbacks(self, on_settings: Callable | None = None, on_flag: Callable | None = None): + self._on_settings_click = on_settings + self._on_flag_click = on_flag + + def render(self, rect: rl.Rectangle): + self.update_state() + + # Background + rl.draw_rectangle_rec(rect, Colors.SIDEBAR_BG) + + self._draw_buttons(rect) + self._draw_network_indicator(rect) + self._draw_metrics(rect) + + self._handle_mouse_release() + + def update_state(self): + sm = ui_state.sm + if not sm.updated['deviceState']: + return + + device_state = sm['deviceState'] + + self._update_network_status(device_state) + self._update_temperature_status(device_state) + self._update_connection_status(device_state) + self._update_panda_status() + + def _update_network_status(self, device_state): + self._net_type = NETWORK_TYPES.get(device_state.networkType.raw, "Unknown") + strength = device_state.networkStrength + self._net_strength = max(0, min(5, strength.raw + 1)) if strength > 0 else 0 + + def _update_temperature_status(self, device_state): + thermal_status = device_state.thermalStatus + + if thermal_status == ThermalStatus.green: + self._temp_status.update("TEMP", "GOOD", Colors.GOOD) + elif thermal_status == ThermalStatus.yellow: + self._temp_status.update("TEMP", "OK", Colors.WARNING) + else: + self._temp_status.update("TEMP", "HIGH", Colors.DANGER) + + def _update_connection_status(self, device_state): + last_ping = device_state.lastAthenaPingTime + if last_ping == 0: + self._connect_status.update("CONNECT", "OFFLINE", Colors.WARNING) + elif time.monotonic_ns() - last_ping < 80_000_000_000: # 80 seconds in nanoseconds + self._connect_status.update("CONNECT", "ONLINE", Colors.GOOD) + else: + self._connect_status.update("CONNECT", "ERROR", Colors.DANGER) + + def _update_panda_status(self): + if ui_state.panda_type == log.PandaState.PandaType.unknown: + self._panda_status.update("NO", "PANDA", Colors.DANGER) + else: + self._panda_status.update("VEHICLE", "ONLINE", Colors.GOOD) + + def _handle_mouse_release(self): + if not rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT): + return + + mouse_pos = rl.get_mouse_position() + if rl.check_collision_point_rec(mouse_pos, SETTINGS_BTN): + if self._on_settings_click: + self._on_settings_click() + elif rl.check_collision_point_rec(mouse_pos, HOME_BTN) and ui_state.started: + if self._on_flag_click: + self._on_flag_click() + + def _draw_buttons(self, rect: rl.Rectangle): + mouse_pos = rl.get_mouse_position() + mouse_down = rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT) + + + # Settings button + settings_down = mouse_down and rl.check_collision_point_rec(mouse_pos, SETTINGS_BTN) + tint = Colors.BUTTON_PRESSED if settings_down else Colors.BUTTON_NORMAL + rl.draw_texture(self._settings_img, int(SETTINGS_BTN.x), int(SETTINGS_BTN.y), tint) + + # Home/Flag button + flag_pressed = mouse_down and rl.check_collision_point_rec(mouse_pos, HOME_BTN) + button_img = self._flag_img if ui_state.started else self._home_img + + tint = Colors.BUTTON_PRESSED if (ui_state.started and flag_pressed) else Colors.BUTTON_NORMAL + rl.draw_texture(button_img, int(HOME_BTN.x), int(HOME_BTN.y), tint) + + def _draw_network_indicator(self, rect: rl.Rectangle): + # Signal strength dots + x_start = rect.x + 58 + y_pos = rect.y + 196 + dot_size = 27 + dot_spacing = 37 + + for i in range(5): + color = Colors.WHITE if i < self._net_strength else Colors.GRAY + x = int(x_start + i * dot_spacing + dot_size // 2) + y = int(y_pos + dot_size // 2) + rl.draw_circle(x, y, dot_size // 2, color) + + # Network type text + text_y = rect.y + 247 + text_pos = rl.Vector2(rect.x + 58, text_y) + rl.draw_text_ex(self._font_regular, self._net_type, text_pos, 35, 0, Colors.WHITE) + + def _draw_metrics(self, rect: rl.Rectangle): + metrics = [(self._temp_status, 338), (self._panda_status, 496), (self._connect_status, 654)] + + for metric, y_offset in metrics: + self._draw_metric(rect, metric, rect.y + y_offset) + + def _draw_metric(self, rect: rl.Rectangle, metric: MetricData, y: float): + metric_rect = rl.Rectangle(rect.x + METRIC_MARGIN, y, METRIC_WIDTH, METRIC_HEIGHT) + # Draw colored left edge (clipped rounded rectangle) + edge_rect = rl.Rectangle(metric_rect.x + 4, metric_rect.y + 4, 100, 118) + rl.begin_scissor_mode(int(metric_rect.x + 4), int(metric_rect.y), 18, int(metric_rect.height)) + rl.draw_rectangle_rounded(edge_rect, 0.18, 10, metric.color) + rl.end_scissor_mode() + + # Draw border + rl.draw_rectangle_rounded_lines_ex(metric_rect, 0.15, 10, 2, Colors.METRIC_BORDER) + + # Draw text + text = f"{metric.label}\n{metric.value}" + text_size = rl.measure_text_ex(self._font_bold, text, 35, 0) + text_pos = rl.Vector2( + metric_rect.x + 22 + (metric_rect.width - 22 - text_size.x) / 2, + metric_rect.y + (metric_rect.height - text_size.y) / 2 + ) + rl.draw_text_ex(self._font_bold, text, text_pos, 35, 0, Colors.WHITE) diff --git a/selfdrive/ui/onroad/augmented_road_view.py b/selfdrive/ui/onroad/augmented_road_view.py index f987d70c9c..fa7469d2dd 100644 --- a/selfdrive/ui/onroad/augmented_road_view.py +++ b/selfdrive/ui/onroad/augmented_road_view.py @@ -26,7 +26,7 @@ BORDER_COLORS = { class AugmentedRoadView(CameraView): - def __init__(self, stream_type: VisionStreamType): + def __init__(self, stream_type: VisionStreamType = VisionStreamType.VISION_STREAM_ROAD): super().__init__("camerad", stream_type) self.device_camera: DeviceCameraConfig | None = None diff --git a/selfdrive/ui/ui.py b/selfdrive/ui/ui.py new file mode 100755 index 0000000000..8a3e34466f --- /dev/null +++ b/selfdrive/ui/ui.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +import pyray as rl +from openpilot.system.ui.lib.application import gui_app +from openpilot.selfdrive.ui.layouts.main import MainLayout +from openpilot.selfdrive.ui.ui_state import ui_state + + +def main(): + gui_app.init_window("UI") + main_layout = MainLayout() + for _ in gui_app.render(): + ui_state.update() + + #TODO handle brigntness and awake state here + + main_layout.render(rl.Rectangle(0, 0, gui_app.width, gui_app.height)) + + +if __name__ == "__main__": + main()