ui: add main UI entry point (#35422)

* add new main UI entry point

* cleanup

* mv to selfdrive/ui

* fix imports

* handle_mouse_click

* use ui_state

* remove ui_state from gui_app

* setup callbacks

* handle clicks

* put layouts in a dict

* update state in render

* rebase master

* implement settings sidebar

* rename files

selfdrive/ui/layouts/home.py

@@ -0,0 +1,17 @@
+import pyray as rl
+from openpilot.system.ui.lib.label import gui_text_box
+
+
+class HomeLayout:
+  def __init__(self):
+    pass
+
+  def render(self, rect: rl.Rectangle):
+    gui_text_box(
+      rect,
+      "Demo Home Layout",
+      font_size=170,
+      color=rl.WHITE,
+      alignment=rl.GuiTextAlignment.TEXT_ALIGN_CENTER,
+      alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE,
+    )

selfdrive/ui/layouts/main.py

@@ -0,0 +1,92 @@
+import pyray as rl
+from enum import IntEnum
+from openpilot.selfdrive.ui.layouts.sidebar import Sidebar, SIDEBAR_WIDTH
+from openpilot.selfdrive.ui.layouts.home import HomeLayout
+from openpilot.selfdrive.ui.layouts.settings.settings import SettingsLayout
+from openpilot.selfdrive.ui.ui_state import ui_state
+from openpilot.selfdrive.ui.onroad.augmented_road_view import AugmentedRoadView
+
+
+class MainState(IntEnum):
+  HOME = 0
+  SETTINGS = 1
+  ONROAD = 2
+
+
+class MainLayout:
+  def __init__(self):
+    self._sidebar = Sidebar()
+    self._sidebar_visible = True
+    self._current_mode = MainState.HOME
+    self._prev_onroad = False
+    self._window_rect = None
+    self._current_callback: callable | None = None
+
+    # Initialize layouts
+    self._layouts = {MainState.HOME: HomeLayout(), MainState.SETTINGS: SettingsLayout(), MainState.ONROAD: AugmentedRoadView()}
+
+    self._sidebar_rect = rl.Rectangle(0, 0, 0, 0)
+    self._content_rect = rl.Rectangle(0, 0, 0, 0)
+
+    # Set callbacks
+    self._setup_callbacks()
+
+  def render(self, rect):
+    self._current_callback = None
+
+    self._update_layout_rects(rect)
+    self._render_main_content()
+    self._handle_input()
+
+    if self._current_callback:
+      self._current_callback()
+
+  def _setup_callbacks(self):
+    self._sidebar.set_callbacks(
+      on_settings=lambda: setattr(self, '_current_callback', self._on_settings_clicked),
+      on_flag=lambda: setattr(self, '_current_callback', self._on_flag_clicked),
+    )
+    self._layouts[MainState.SETTINGS].set_callbacks(
+      on_close=lambda: setattr(self, '_current_callback', self._on_settings_closed)
+    )
+
+  def _update_layout_rects(self, rect):
+    self._window_rect = rect
+    self._sidebar_rect = rl.Rectangle(rect.x, rect.y, SIDEBAR_WIDTH, rect.height)
+
+    x_offset = SIDEBAR_WIDTH if self._sidebar_visible else 0
+    self._content_rect = rl.Rectangle(rect.y + x_offset, rect.y, rect.width - x_offset, rect.height)
+
+  def _on_settings_clicked(self):
+    self._current_mode = MainState.SETTINGS
+    self._sidebar_visible = False
+
+  def _on_settings_closed(self):
+    self._current_mode = MainState.HOME if not ui_state.started else MainState.ONROAD
+    self._sidebar_visible = True
+
+  def _on_flag_clicked(self):
+    pass
+
+  def _render_main_content(self):
+    # Render sidebar
+    if self._sidebar_visible:
+      self._sidebar.render(self._sidebar_rect)
+
+    if ui_state.started != self._prev_onroad:
+      self._prev_onroad = ui_state.started
+      if ui_state.started:
+        self._current_mode = MainState.ONROAD
+      else:
+        self._current_mode = MainState.HOME
+
+    content_rect = self._content_rect if self._sidebar_visible else self._window_rect
+    self._layouts[self._current_mode].render(content_rect)
+
+  def _handle_input(self):
+    if self._current_mode != MainState.ONROAD or not rl.is_mouse_button_pressed(rl.MouseButton.MOUSE_BUTTON_LEFT):
+      return
+
+    mouse_pos = rl.get_mouse_position()
+    if rl.check_collision_point_rec(mouse_pos, self._content_rect):
+      self._sidebar_visible = not self._sidebar_visible

selfdrive/ui/layouts/settings/settings.py

@@ -0,0 +1,174 @@
+import pyray as rl
+from dataclasses import dataclass
+from enum import IntEnum
+from collections.abc import Callable
+from openpilot.common.params import Params
+from openpilot.system.ui.lib.application import gui_app, FontWeight
+from openpilot.system.ui.lib.label import gui_text_box
+
+# Import individual panels
+
+SETTINGS_CLOSE_TEXT = "X"
+# Constants
+SIDEBAR_WIDTH = 500
+CLOSE_BTN_SIZE = 200
+NAV_BTN_HEIGHT = 80
+PANEL_MARGIN = 50
+SCROLL_SPEED = 30
+
+# Colors
+SIDEBAR_COLOR = rl.BLACK
+PANEL_COLOR = rl.Color(41, 41, 41, 255)
+CLOSE_BTN_COLOR = rl.Color(41, 41, 41, 255)
+CLOSE_BTN_PRESSED = rl.Color(59, 59, 59, 255)
+TEXT_NORMAL = rl.Color(128, 128, 128, 255)
+TEXT_SELECTED = rl.Color(255, 255, 255, 255)
+TEXT_PRESSED = rl.Color(173, 173, 173, 255)
+
+
+class PanelType(IntEnum):
+  DEVICE = 0
+  NETWORK = 1
+  TOGGLES = 2
+  SOFTWARE = 3
+  FIREHOSE = 4
+  DEVELOPER = 5
+
+
+@dataclass
+class PanelInfo:
+  name: str
+  instance: object
+  button_rect: rl.Rectangle
+
+
+class SettingsLayout:
+  def __init__(self):
+    self._params = Params()
+    self._current_panel = PanelType.DEVICE
+    self._close_btn_pressed = False
+    self._scroll_offset = 0.0
+    self._max_scroll = 0.0
+
+    # Panel configuration
+    self._panels = {
+      PanelType.DEVICE: PanelInfo("Device", None, rl.Rectangle(0, 0, 0, 0)),
+      PanelType.TOGGLES: PanelInfo("Toggles", None, rl.Rectangle(0, 0, 0, 0)),
+      PanelType.SOFTWARE: PanelInfo("Software", None, rl.Rectangle(0, 0, 0, 0)),
+      PanelType.FIREHOSE: PanelInfo("Firehose", None, rl.Rectangle(0, 0, 0, 0)),
+      PanelType.NETWORK: PanelInfo("Network", None, rl.Rectangle(0, 0, 0, 0)),
+      PanelType.DEVELOPER: PanelInfo("Developer", None, rl.Rectangle(0, 0, 0, 0)),
+    }
+
+    self._font_medium = gui_app.font(FontWeight.MEDIUM)
+    self._font_bold = gui_app.font(FontWeight.SEMI_BOLD)
+
+    # Callbacks
+    self._close_callback: Callable | None = None
+
+  def set_callbacks(self, on_close: Callable):
+    self._close_callback = on_close
+
+  def render(self, rect: rl.Rectangle):
+    # Calculate layout
+    sidebar_rect = rl.Rectangle(rect.x, rect.y, SIDEBAR_WIDTH, rect.height)
+    panel_rect = rl.Rectangle(rect.x + SIDEBAR_WIDTH, rect.y, rect.width - SIDEBAR_WIDTH, rect.height)
+
+    # Draw components
+    self._draw_sidebar(sidebar_rect)
+    self._draw_current_panel(panel_rect)
+
+    if rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT):
+      self.handle_mouse_release(rl.get_mouse_position())
+
+  def _draw_sidebar(self, rect: rl.Rectangle):
+    rl.draw_rectangle_rec(rect, SIDEBAR_COLOR)
+
+    # Close button
+    close_btn_rect = rl.Rectangle(
+      rect.x + (rect.width - CLOSE_BTN_SIZE) / 2, rect.y + 45, CLOSE_BTN_SIZE, CLOSE_BTN_SIZE
+    )
+
+    close_color = CLOSE_BTN_PRESSED if self._close_btn_pressed else CLOSE_BTN_COLOR
+    rl.draw_rectangle_rounded(close_btn_rect, 0.5, 20, close_color)
+    close_text_size = rl.measure_text_ex(self._font_bold, SETTINGS_CLOSE_TEXT, 140, 0)
+    close_text_pos = rl.Vector2(
+      close_btn_rect.x + (close_btn_rect.width - close_text_size.x) / 2,
+      close_btn_rect.y + (close_btn_rect.height - close_text_size.y) / 2 - 20,
+    )
+    rl.draw_text_ex(self._font_bold, SETTINGS_CLOSE_TEXT, close_text_pos, 140, 0, TEXT_SELECTED)
+
+    # Store close button rect for click detection
+    self._close_btn_rect = close_btn_rect
+
+    # Navigation buttons
+    nav_start_y = rect.y + 300
+    button_spacing = 20
+
+    i = 0
+    for panel_type, panel_info in self._panels.items():
+      button_rect = rl.Rectangle(
+        rect.x + 50,
+        nav_start_y + i * (NAV_BTN_HEIGHT + button_spacing),
+        rect.width - 150,  # Right-aligned with margin
+        NAV_BTN_HEIGHT,
+      )
+
+      # Button styling
+      is_selected = panel_type == self._current_panel
+      text_color = TEXT_SELECTED if is_selected else TEXT_NORMAL
+
+      # Draw button text (right-aligned)
+      text_size = rl.measure_text_ex(self._font_medium, panel_info.name, 65, 0)
+      text_pos = rl.Vector2(
+        button_rect.x + button_rect.width - text_size.x, button_rect.y + (button_rect.height - text_size.y) / 2
+      )
+      rl.draw_text_ex(self._font_medium, panel_info.name, text_pos, 65, 0, text_color)
+
+      # Store button rect for click detection
+      panel_info.button_rect = button_rect
+      i += 1
+
+  def _draw_current_panel(self, rect: rl.Rectangle):
+    content_rect = rl.Rectangle(rect.x + PANEL_MARGIN, rect.y + 25, rect.width - (PANEL_MARGIN * 2), rect.height - 50)
+    rl.draw_rectangle_rounded(content_rect, 0.03, 30, PANEL_COLOR)
+    gui_text_box(
+      content_rect,
+      f"Demo {self._panels[self._current_panel].name} Panel",
+      font_size=170,
+      color=rl.WHITE,
+      alignment=rl.GuiTextAlignment.TEXT_ALIGN_CENTER,
+      alignment_vertical=rl.GuiTextAlignmentVertical.TEXT_ALIGN_MIDDLE,
+    )
+
+  def handle_mouse_release(self, mouse_pos: rl.Vector2) -> bool:
+    # Check close button
+    if rl.check_collision_point_rec(mouse_pos, self._close_btn_rect):
+      self._close_btn_pressed = True
+      if self._close_callback:
+        self._close_callback()
+      return True
+
+    # Check navigation buttons
+    for panel_type, panel_info in self._panels.items():
+      if rl.check_collision_point_rec(mouse_pos, panel_info.button_rect):
+        self._switch_to_panel(panel_type)
+        return True
+
+    return False
+
+  def _switch_to_panel(self, panel_type: PanelType):
+    if panel_type != self._current_panel:
+      self._current_panel = panel_type
+      self._scroll_offset = 0.0  # Reset scroll when switching panels
+      self._transition_progress = 0.0
+      self._transitioning = True
+
+  def set_current_panel(self, index: int, param: str = ""):
+    panel_types = list(self._panels.keys())
+    if 0 <= index < len(panel_types):
+      self._switch_to_panel(panel_types[index])
+
+  def close_settings(self):
+    if self._close_callback:
+      self._close_callback()

selfdrive/ui/layouts/sidebar.py

@@ -0,0 +1,207 @@
+import pyray as rl
+import time
+from dataclasses import dataclass
+from collections.abc import Callable
+from cereal import log
+from openpilot.selfdrive.ui.ui_state import ui_state
+from openpilot.system.ui.lib.application import gui_app, FontWeight
+
+SIDEBAR_WIDTH = 300
+METRIC_HEIGHT = 126
+METRIC_WIDTH = 240
+METRIC_MARGIN = 30
+
+SETTINGS_BTN = rl.Rectangle(50, 35, 200, 117)
+HOME_BTN = rl.Rectangle(60, 860, 180, 180)
+
+ThermalStatus = log.DeviceState.ThermalStatus
+NetworkType = log.DeviceState.NetworkType
+
+# Color scheme
+class Colors:
+  SIDEBAR_BG = rl.Color(57, 57, 57, 255)
+  WHITE = rl.Color(255, 255, 255, 255)
+  WHITE_DIM = rl.Color(255, 255, 255, 85)
+  GRAY = rl.Color(84, 84, 84, 255)
+
+  # Status colors
+  GOOD = rl.Color(255, 255, 255, 255)
+  WARNING = rl.Color(218, 202, 37, 255)
+  DANGER = rl.Color(201, 34, 49, 255)
+
+  # UI elements
+  METRIC_BORDER = rl.Color(255, 255, 255, 85)
+  BUTTON_NORMAL = rl.Color(255, 255, 255, 255)
+  BUTTON_PRESSED = rl.Color(255, 255, 255, 166)
+
+NETWORK_TYPES = {
+  NetworkType.none: "Offline",
+  NetworkType.wifi: "WiFi",
+  NetworkType.cell2G: "2G",
+  NetworkType.cell3G: "3G",
+  NetworkType.cell4G: "LTE",
+  NetworkType.cell5G: "5G",
+  NetworkType.ethernet: "Ethernet",
+}
+
+
+@dataclass(slots=True)
+class MetricData:
+  label: str
+  value: str
+  color: rl.Color
+
+  def update(self, label: str, value: str, color: rl.Color):
+    self.label = label
+    self.value = value
+    self.color = color
+
+class Sidebar:
+  def __init__(self):
+    self._net_type = NETWORK_TYPES.get(NetworkType.none)
+    self._net_strength = 0
+
+    self._temp_status = MetricData("TEMP", "GOOD", Colors.GOOD)
+    self._panda_status = MetricData("VEHICLE", "ONLINE", Colors.GOOD)
+    self._connect_status = MetricData("CONNECT", "OFFLINE", Colors.WARNING)
+
+    self._home_img = gui_app.texture("images/button_home.png", HOME_BTN.width, HOME_BTN.height)
+    self._flag_img = gui_app.texture("images/button_flag.png", HOME_BTN.width, HOME_BTN.height)
+    self._settings_img = gui_app.texture("images/button_settings.png", SETTINGS_BTN.width, SETTINGS_BTN.height)
+    self._font_regular = gui_app.font(FontWeight.NORMAL)
+    self._font_bold = gui_app.font(FontWeight.SEMI_BOLD)
+
+     # Callbacks
+    self._on_settings_click: Callable | None = None
+    self._on_flag_click: Callable | None = None
+
+  def set_callbacks(self, on_settings: Callable | None = None, on_flag: Callable | None = None):
+    self._on_settings_click = on_settings
+    self._on_flag_click = on_flag
+
+  def render(self, rect: rl.Rectangle):
+    self.update_state()
+
+    # Background
+    rl.draw_rectangle_rec(rect, Colors.SIDEBAR_BG)
+
+    self._draw_buttons(rect)
+    self._draw_network_indicator(rect)
+    self._draw_metrics(rect)
+
+    self._handle_mouse_release()
+
+  def update_state(self):
+    sm = ui_state.sm
+    if not sm.updated['deviceState']:
+      return
+
+    device_state = sm['deviceState']
+
+    self._update_network_status(device_state)
+    self._update_temperature_status(device_state)
+    self._update_connection_status(device_state)
+    self._update_panda_status()
+
+  def _update_network_status(self, device_state):
+    self._net_type = NETWORK_TYPES.get(device_state.networkType.raw, "Unknown")
+    strength = device_state.networkStrength
+    self._net_strength = max(0, min(5, strength.raw + 1)) if strength > 0 else 0
+
+  def _update_temperature_status(self, device_state):
+    thermal_status = device_state.thermalStatus
+
+    if thermal_status == ThermalStatus.green:
+      self._temp_status.update("TEMP", "GOOD", Colors.GOOD)
+    elif thermal_status == ThermalStatus.yellow:
+      self._temp_status.update("TEMP", "OK", Colors.WARNING)
+    else:
+      self._temp_status.update("TEMP", "HIGH", Colors.DANGER)
+
+  def _update_connection_status(self, device_state):
+    last_ping = device_state.lastAthenaPingTime
+    if last_ping == 0:
+      self._connect_status.update("CONNECT", "OFFLINE", Colors.WARNING)
+    elif time.monotonic_ns() - last_ping < 80_000_000_000:  # 80 seconds in nanoseconds
+      self._connect_status.update("CONNECT", "ONLINE", Colors.GOOD)
+    else:
+      self._connect_status.update("CONNECT", "ERROR", Colors.DANGER)
+
+  def _update_panda_status(self):
+    if ui_state.panda_type == log.PandaState.PandaType.unknown:
+      self._panda_status.update("NO", "PANDA", Colors.DANGER)
+    else:
+      self._panda_status.update("VEHICLE", "ONLINE", Colors.GOOD)
+
+  def _handle_mouse_release(self):
+    if not rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT):
+      return
+
+    mouse_pos = rl.get_mouse_position()
+    if rl.check_collision_point_rec(mouse_pos, SETTINGS_BTN):
+      if self._on_settings_click:
+        self._on_settings_click()
+    elif rl.check_collision_point_rec(mouse_pos, HOME_BTN) and ui_state.started:
+      if self._on_flag_click:
+        self._on_flag_click()
+
+  def _draw_buttons(self, rect: rl.Rectangle):
+    mouse_pos = rl.get_mouse_position()
+    mouse_down = rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT)
+
+
+    # Settings button
+    settings_down = mouse_down and rl.check_collision_point_rec(mouse_pos, SETTINGS_BTN)
+    tint = Colors.BUTTON_PRESSED if settings_down else Colors.BUTTON_NORMAL
+    rl.draw_texture(self._settings_img, int(SETTINGS_BTN.x), int(SETTINGS_BTN.y), tint)
+
+    # Home/Flag button
+    flag_pressed = mouse_down and rl.check_collision_point_rec(mouse_pos, HOME_BTN)
+    button_img = self._flag_img if ui_state.started else self._home_img
+
+    tint = Colors.BUTTON_PRESSED if (ui_state.started and flag_pressed) else Colors.BUTTON_NORMAL
+    rl.draw_texture(button_img, int(HOME_BTN.x), int(HOME_BTN.y), tint)
+
+  def _draw_network_indicator(self, rect: rl.Rectangle):
+    # Signal strength dots
+    x_start = rect.x + 58
+    y_pos = rect.y + 196
+    dot_size = 27
+    dot_spacing = 37
+
+    for i in range(5):
+      color = Colors.WHITE if i < self._net_strength else Colors.GRAY
+      x = int(x_start + i * dot_spacing + dot_size // 2)
+      y = int(y_pos + dot_size // 2)
+      rl.draw_circle(x, y, dot_size // 2, color)
+
+    # Network type text
+    text_y = rect.y + 247
+    text_pos = rl.Vector2(rect.x + 58, text_y)
+    rl.draw_text_ex(self._font_regular, self._net_type, text_pos, 35, 0, Colors.WHITE)
+
+  def _draw_metrics(self, rect: rl.Rectangle):
+    metrics = [(self._temp_status, 338), (self._panda_status, 496), (self._connect_status, 654)]
+
+    for metric, y_offset in metrics:
+      self._draw_metric(rect, metric, rect.y + y_offset)
+
+  def _draw_metric(self, rect: rl.Rectangle, metric: MetricData, y: float):
+    metric_rect = rl.Rectangle(rect.x + METRIC_MARGIN, y, METRIC_WIDTH, METRIC_HEIGHT)
+    # Draw colored left edge (clipped rounded rectangle)
+    edge_rect = rl.Rectangle(metric_rect.x + 4, metric_rect.y + 4, 100, 118)
+    rl.begin_scissor_mode(int(metric_rect.x + 4), int(metric_rect.y), 18, int(metric_rect.height))
+    rl.draw_rectangle_rounded(edge_rect, 0.18, 10, metric.color)
+    rl.end_scissor_mode()
+
+    # Draw border
+    rl.draw_rectangle_rounded_lines_ex(metric_rect, 0.15, 10, 2, Colors.METRIC_BORDER)
+
+    # Draw text
+    text = f"{metric.label}\n{metric.value}"
+    text_size = rl.measure_text_ex(self._font_bold, text, 35, 0)
+    text_pos = rl.Vector2(
+      metric_rect.x + 22 + (metric_rect.width - 22 - text_size.x) / 2,
+      metric_rect.y + (metric_rect.height - text_size.y) / 2
+    )
+    rl.draw_text_ex(self._font_bold, text, text_pos, 35, 0, Colors.WHITE)

selfdrive/ui/onroad/augmented_road_view.py

@@ -26,7 +26,7 @@ BORDER_COLORS = {
 
 
 class AugmentedRoadView(CameraView):
-  def __init__(self, stream_type: VisionStreamType):
+  def __init__(self, stream_type: VisionStreamType = VisionStreamType.VISION_STREAM_ROAD):
     super().__init__("camerad", stream_type)
 
     self.device_camera: DeviceCameraConfig | None = None

selfdrive/ui/ui.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+import pyray as rl
+from openpilot.system.ui.lib.application import gui_app
+from openpilot.selfdrive.ui.layouts.main import MainLayout
+from openpilot.selfdrive.ui.ui_state import ui_state
+
+
+def main():
+  gui_app.init_window("UI")
+  main_layout = MainLayout()
+  for _ in gui_app.render():
+    ui_state.update()
+
+    #TODO handle brigntness and awake state here
+
+    main_layout.render(rl.Rectangle(0, 0, gui_app.width, gui_app.height))
+
+
+if __name__ == "__main__":
+  main()