From 13c5c4dacce48f47f60025bc29ac67a44e3038a2 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Wed, 9 Jul 2025 21:33:19 -0700 Subject: [PATCH] raylib: don't drop touch events on device (#35672) * mouse thread * instanciate mouse * type that * pc handling * use mouse event list in widget * use events in scroll panel * no stop that * hack for now * typing * run * clean up --- selfdrive/ui/layouts/settings/settings.py | 4 +- selfdrive/ui/layouts/sidebar.py | 4 +- system/ui/lib/application.py | 97 +++++++++++++++++++++-- system/ui/lib/list_view.py | 4 +- system/ui/lib/scroll_panel.py | 39 +++++---- system/ui/lib/toggle.py | 3 +- system/ui/lib/widget.py | 23 +++--- 7 files changed, 134 insertions(+), 40 deletions(-) diff --git a/selfdrive/ui/layouts/settings/settings.py b/selfdrive/ui/layouts/settings/settings.py index 4a29f408ee..d4dc420b7f 100644 --- a/selfdrive/ui/layouts/settings/settings.py +++ b/selfdrive/ui/layouts/settings/settings.py @@ -7,7 +7,7 @@ from openpilot.selfdrive.ui.layouts.settings.device import DeviceLayout from openpilot.selfdrive.ui.layouts.settings.firehose import FirehoseLayout from openpilot.selfdrive.ui.layouts.settings.software import SoftwareLayout from openpilot.selfdrive.ui.layouts.settings.toggles import TogglesLayout -from openpilot.system.ui.lib.application import gui_app, FontWeight +from openpilot.system.ui.lib.application import gui_app, FontWeight, MousePos from openpilot.system.ui.lib.text_measure import measure_text_cached from openpilot.selfdrive.ui.layouts.network import NetworkLayout from openpilot.system.ui.lib.widget import Widget @@ -132,7 +132,7 @@ class SettingsLayout(Widget): if panel.instance: panel.instance.render(content_rect) - def _handle_mouse_release(self, mouse_pos: rl.Vector2) -> bool: + def _handle_mouse_release(self, mouse_pos: MousePos) -> bool: # Check close button if rl.check_collision_point_rec(mouse_pos, self._close_btn_rect): if self._close_callback: diff --git a/selfdrive/ui/layouts/sidebar.py b/selfdrive/ui/layouts/sidebar.py index 13f69b79d4..cb8c6d472e 100644 --- a/selfdrive/ui/layouts/sidebar.py +++ b/selfdrive/ui/layouts/sidebar.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from collections.abc import Callable from cereal import log from openpilot.selfdrive.ui.ui_state import ui_state -from openpilot.system.ui.lib.application import gui_app, FontWeight +from openpilot.system.ui.lib.application import gui_app, FontWeight, MousePos from openpilot.system.ui.lib.text_measure import measure_text_cached from openpilot.system.ui.lib.widget import Widget @@ -135,7 +135,7 @@ class Sidebar(Widget): else: self._panda_status.update("VEHICLE", "ONLINE", Colors.GOOD) - def _handle_mouse_release(self, mouse_pos: rl.Vector2): + def _handle_mouse_release(self, mouse_pos: MousePos): if rl.check_collision_point_rec(mouse_pos, SETTINGS_BTN): if self._on_settings_click: self._on_settings_click() diff --git a/system/ui/lib/application.py b/system/ui/lib/application.py index ff81fc69ad..12d825b99f 100644 --- a/system/ui/lib/application.py +++ b/system/ui/lib/application.py @@ -3,18 +3,22 @@ import cffi import os import time import pyray as rl +import threading from collections.abc import Callable from collections import deque from dataclasses import dataclass from enum import IntEnum +from typing import NamedTuple from importlib.resources import as_file, files from openpilot.common.swaglog import cloudlog -from openpilot.system.hardware import HARDWARE +from openpilot.system.hardware import HARDWARE, PC +from openpilot.common.realtime import Ratekeeper DEFAULT_FPS = int(os.getenv("FPS", "60")) FPS_LOG_INTERVAL = 5 # Seconds between logging FPS drops FPS_DROP_THRESHOLD = 0.9 # FPS drop threshold for triggering a warning FPS_CRITICAL_THRESHOLD = 0.5 # Critical threshold for triggering strict actions +MOUSE_THREAD_RATE = 140 # touch controller runs at 140Hz ENABLE_VSYNC = os.getenv("ENABLE_VSYNC", "0") == "1" SHOW_FPS = os.getenv("SHOW_FPS") == "1" @@ -47,6 +51,68 @@ class ModalOverlay: callback: Callable | None = None +class MousePos(NamedTuple): + x: float + y: float + + +class MouseEvent(NamedTuple): + pos: MousePos + left_pressed: bool + left_released: bool + left_down: bool + t: float + + +class MouseState: + def __init__(self): + self._events: deque[MouseEvent] = deque(maxlen=MOUSE_THREAD_RATE) # bound event list + self._prev_mouse_event: MouseEvent | None = None + + self._rk = Ratekeeper(MOUSE_THREAD_RATE) + self._lock = threading.Lock() + self._exit_event = threading.Event() + self._thread = None + + def get_events(self) -> list[MouseEvent]: + with self._lock: + events = list(self._events) + self._events.clear() + return events + + def start(self): + self._exit_event.clear() + if self._thread is None or not self._thread.is_alive(): + self._thread = threading.Thread(target=self._run_thread, daemon=True) + self._thread.start() + + def stop(self): + self._exit_event.set() + if self._thread is not None and self._thread.is_alive(): + self._thread.join() + + def _run_thread(self): + while not self._exit_event.is_set(): + rl.poll_input_events() + self._handle_mouse_event() + self._rk.keep_time() + + def _handle_mouse_event(self): + mouse_pos = rl.get_mouse_position() + ev = MouseEvent( + MousePos(mouse_pos.x, mouse_pos.y), + rl.is_mouse_button_pressed(rl.MouseButton.MOUSE_BUTTON_LEFT), + rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT), + rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT), + time.monotonic(), + ) + # Only add changes + if self._prev_mouse_event is None or ev[:-1] != self._prev_mouse_event[:-1]: + with self._lock: + self._events.append(ev) + self._prev_mouse_event = ev + + class GuiApplication: def __init__(self, width: int, height: int): self._fonts: dict[FontWeight, rl.Font] = {} @@ -63,8 +129,11 @@ class GuiApplication: self._trace_log_callback = None self._modal_overlay = ModalOverlay() + self._mouse = MouseState() + self._mouse_events: list[MouseEvent] = [] + # Debug variables - self._mouse_history: deque[rl.Vector2] = deque(maxlen=DEFAULT_FPS * 2) + self._mouse_history: deque[MousePos] = deque(maxlen=MOUSE_THREAD_RATE) def request_close(self): self._window_close_requested = True @@ -94,6 +163,9 @@ class GuiApplication: self._set_styles() self._load_fonts() + if not PC: + self._mouse.start() + def set_modal_overlay(self, overlay, callback: Callable | None = None): self._modal_overlay = ModalOverlay(overlay=overlay, callback=callback) @@ -154,11 +226,25 @@ class GuiApplication: rl.unload_render_texture(self._render_texture) self._render_texture = None + if not PC: + self._mouse.stop() + rl.close_window() + @property + def mouse_events(self) -> list[MouseEvent]: + return self._mouse_events + def render(self): try: while not (self._window_close_requested or rl.window_should_close()): + if PC: + # Thread is not used on PC, need to manually add mouse events + self._mouse._handle_mouse_event() + + # Store all mouse events for the current frame + self._mouse_events = self._mouse.get_events() + if self._render_texture: rl.begin_texture_mode(self._render_texture) rl.clear_background(rl.BLACK) @@ -196,9 +282,10 @@ class GuiApplication: rl.draw_fps(10, 10) if SHOW_TOUCHES: - if rl.is_mouse_button_pressed(rl.MouseButton.MOUSE_BUTTON_LEFT): - self._mouse_history.clear() - self._mouse_history.append(rl.get_mouse_position()) + for mouse_event in self._mouse_events: + if mouse_event.left_pressed: + self._mouse_history.clear() + self._mouse_history.append(mouse_event.pos) if self._mouse_history: mouse_pos = self._mouse_history[-1] diff --git a/system/ui/lib/list_view.py b/system/ui/lib/list_view.py index f5581be23e..4b6f434de3 100644 --- a/system/ui/lib/list_view.py +++ b/system/ui/lib/list_view.py @@ -2,7 +2,7 @@ import os import pyray as rl from collections.abc import Callable from abc import ABC -from openpilot.system.ui.lib.application import gui_app, FontWeight +from openpilot.system.ui.lib.application import gui_app, FontWeight, MousePos from openpilot.system.ui.lib.text_measure import measure_text_cached from openpilot.system.ui.lib.wrap_text import wrap_text from openpilot.system.ui.lib.button import gui_button, ButtonStyle @@ -229,7 +229,7 @@ class ListItem(Widget): super().set_parent_rect(parent_rect) self._rect.width = parent_rect.width - def _handle_mouse_release(self, mouse_pos: rl.Vector2): + def _handle_mouse_release(self, mouse_pos: MousePos): if not self.is_visible: return diff --git a/system/ui/lib/scroll_panel.py b/system/ui/lib/scroll_panel.py index 32f3b7b575..9f01865b9a 100644 --- a/system/ui/lib/scroll_panel.py +++ b/system/ui/lib/scroll_panel.py @@ -1,6 +1,8 @@ +import time import pyray as rl from collections import deque from enum import IntEnum +from openpilot.system.ui.lib.application import gui_app, MouseEvent, MousePos # Scroll constants for smooth scrolling behavior MOUSE_WHEEL_SCROLL_SPEED = 30 @@ -38,51 +40,54 @@ class GuiScrollPanel: self._bounds_rect: rl.Rectangle | None = None def handle_scroll(self, bounds: rl.Rectangle, content: rl.Rectangle) -> rl.Vector2: + # TODO: HACK: this class is driven by mouse events, so we need to ensure we have at least one event to process + for mouse_event in gui_app.mouse_events or [MouseEvent(MousePos(0, 0), False, False, False, time.monotonic())]: + self._handle_mouse_event(mouse_event, bounds, content) + return self._offset + + def _handle_mouse_event(self, mouse_event: MouseEvent, bounds: rl.Rectangle, content: rl.Rectangle): # Store rectangles for reference self._content_rect = content self._bounds_rect = bounds - # Calculate time delta - current_time = rl.get_time() - - mouse_pos = rl.get_mouse_position() max_scroll_y = max(content.height - bounds.height, 0) # Start dragging on mouse press - if rl.check_collision_point_rec(mouse_pos, bounds) and rl.is_mouse_button_pressed(rl.MouseButton.MOUSE_BUTTON_LEFT): + if rl.check_collision_point_rec(mouse_event.pos, bounds) and mouse_event.left_pressed: if self._scroll_state == ScrollState.IDLE or self._scroll_state == ScrollState.BOUNCING: self._scroll_state = ScrollState.DRAGGING_CONTENT if self._show_vertical_scroll_bar: scrollbar_width = rl.gui_get_style(rl.GuiControl.LISTVIEW, rl.GuiListViewProperty.SCROLLBAR_WIDTH) scrollbar_x = bounds.x + bounds.width - scrollbar_width - if mouse_pos.x >= scrollbar_x: + if mouse_event.pos.x >= scrollbar_x: self._scroll_state = ScrollState.DRAGGING_SCROLLBAR # TODO: hacky # when clicking while moving, go straight into dragging self._is_dragging = abs(self._velocity_y) > MIN_VELOCITY - self._last_mouse_y = mouse_pos.y - self._start_mouse_y = mouse_pos.y - self._last_drag_time = current_time + self._last_mouse_y = mouse_event.pos.y + self._start_mouse_y = mouse_event.pos.y + self._last_drag_time = mouse_event.t self._velocity_history.clear() self._velocity_y = 0.0 self._bounce_offset = 0.0 # Handle active dragging if self._scroll_state == ScrollState.DRAGGING_CONTENT or self._scroll_state == ScrollState.DRAGGING_SCROLLBAR: - if rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT): - delta_y = mouse_pos.y - self._last_mouse_y + if mouse_event.left_down: + delta_y = mouse_event.pos.y - self._last_mouse_y # Track velocity for inertia - time_since_last_drag = current_time - self._last_drag_time + time_since_last_drag = mouse_event.t - self._last_drag_time if time_since_last_drag > 0: - drag_velocity = delta_y / time_since_last_drag / 60.0 + # TODO: HACK: /2 since we usually get two touch events per frame + drag_velocity = delta_y / time_since_last_drag / 60.0 / 2 # TODO: shouldn't be hardcoded self._velocity_history.append(drag_velocity) - self._last_drag_time = current_time + self._last_drag_time = mouse_event.t # Detect actual dragging - total_drag = abs(mouse_pos.y - self._start_mouse_y) + total_drag = abs(mouse_event.pos.y - self._start_mouse_y) if total_drag > DRAG_THRESHOLD: self._is_dragging = True @@ -96,9 +101,9 @@ class GuiScrollPanel: scroll_ratio = content.height / bounds.height self._offset.y -= delta_y * scroll_ratio - self._last_mouse_y = mouse_pos.y + self._last_mouse_y = mouse_event.pos.y - elif rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT): + elif mouse_event.left_released: # Calculate flick velocity if self._velocity_history: total_weight = 0 diff --git a/system/ui/lib/toggle.py b/system/ui/lib/toggle.py index 8ed9a655ec..2edf2ff11c 100644 --- a/system/ui/lib/toggle.py +++ b/system/ui/lib/toggle.py @@ -1,4 +1,5 @@ import pyray as rl +from openpilot.system.ui.lib.application import MousePos from openpilot.system.ui.lib.widget import Widget ON_COLOR = rl.Color(51, 171, 76, 255) @@ -23,7 +24,7 @@ class Toggle(Widget): def set_rect(self, rect: rl.Rectangle): self._rect = rl.Rectangle(rect.x, rect.y, WIDTH, HEIGHT) - def _handle_mouse_release(self, mouse_pos: rl.Vector2): + def _handle_mouse_release(self, mouse_pos: MousePos): if not self._enabled: return diff --git a/system/ui/lib/widget.py b/system/ui/lib/widget.py index 3539f5594b..7a436b8e9f 100644 --- a/system/ui/lib/widget.py +++ b/system/ui/lib/widget.py @@ -2,6 +2,7 @@ import abc import pyray as rl from enum import IntEnum from collections.abc import Callable +from openpilot.system.ui.lib.application import gui_app, MousePos class DialogResult(IntEnum): @@ -66,18 +67,18 @@ class Widget(abc.ABC): ret = self._render(self._rect) # Keep track of whether mouse down started within the widget's rectangle - mouse_pos = rl.get_mouse_position() - if rl.is_mouse_button_pressed(rl.MouseButton.MOUSE_BUTTON_LEFT) and self._touch_valid(): - if rl.check_collision_point_rec(mouse_pos, self._rect): - self._is_pressed = True + for mouse_event in gui_app.mouse_events: + if mouse_event.left_pressed and self._touch_valid(): + if rl.check_collision_point_rec(mouse_event.pos, self._rect): + self._is_pressed = True - elif not self._touch_valid(): - self._is_pressed = False + elif not self._touch_valid(): + self._is_pressed = False - elif rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT): - if self._is_pressed and rl.check_collision_point_rec(mouse_pos, self._rect): - self._handle_mouse_release(mouse_pos) - self._is_pressed = False + elif mouse_event.left_released: + if self._is_pressed and rl.check_collision_point_rec(mouse_event.pos, self._rect): + self._handle_mouse_release(mouse_event.pos) + self._is_pressed = False return ret @@ -91,6 +92,6 @@ class Widget(abc.ABC): def _update_layout_rects(self) -> None: """Optionally update any layout rects on Widget rect change.""" - def _handle_mouse_release(self, mouse_pos: rl.Vector2) -> bool: + def _handle_mouse_release(self, mouse_pos: MousePos) -> bool: """Optionally handle mouse release events.""" return False