raylib: don't drop touch events on device (#35672)

* mouse thread

* instanciate mouse

* type that

* pc handling

* use mouse event list in widget

* use events in scroll panel

* no stop that

* hack for now

* typing

* run

* clean up

selfdrive/ui/layouts/settings/settings.py

@@ -7,7 +7,7 @@ from openpilot.selfdrive.ui.layouts.settings.device import DeviceLayout
 from openpilot.selfdrive.ui.layouts.settings.firehose import FirehoseLayout
 from openpilot.selfdrive.ui.layouts.settings.software import SoftwareLayout
 from openpilot.selfdrive.ui.layouts.settings.toggles import TogglesLayout
-from openpilot.system.ui.lib.application import gui_app, FontWeight
+from openpilot.system.ui.lib.application import gui_app, FontWeight, MousePos
 from openpilot.system.ui.lib.text_measure import measure_text_cached
 from openpilot.selfdrive.ui.layouts.network import NetworkLayout
 from openpilot.system.ui.lib.widget import Widget
@@ -132,7 +132,7 @@ class SettingsLayout(Widget):
     if panel.instance:
       panel.instance.render(content_rect)
 
-  def _handle_mouse_release(self, mouse_pos: rl.Vector2) -> bool:
+  def _handle_mouse_release(self, mouse_pos: MousePos) -> bool:
     # Check close button
     if rl.check_collision_point_rec(mouse_pos, self._close_btn_rect):
       if self._close_callback:

selfdrive/ui/layouts/sidebar.py

@@ -4,7 +4,7 @@ from dataclasses import dataclass
 from collections.abc import Callable
 from cereal import log
 from openpilot.selfdrive.ui.ui_state import ui_state
-from openpilot.system.ui.lib.application import gui_app, FontWeight
+from openpilot.system.ui.lib.application import gui_app, FontWeight, MousePos
 from openpilot.system.ui.lib.text_measure import measure_text_cached
 from openpilot.system.ui.lib.widget import Widget
 
@@ -135,7 +135,7 @@ class Sidebar(Widget):
     else:
       self._panda_status.update("VEHICLE", "ONLINE", Colors.GOOD)
 
-  def _handle_mouse_release(self, mouse_pos: rl.Vector2):
+  def _handle_mouse_release(self, mouse_pos: MousePos):
     if rl.check_collision_point_rec(mouse_pos, SETTINGS_BTN):
       if self._on_settings_click:
         self._on_settings_click()

system/ui/lib/application.py

@@ -3,18 +3,22 @@ import cffi
 import os
 import time
 import pyray as rl
+import threading
 from collections.abc import Callable
 from collections import deque
 from dataclasses import dataclass
 from enum import IntEnum
+from typing import NamedTuple
 from importlib.resources import as_file, files
 from openpilot.common.swaglog import cloudlog
-from openpilot.system.hardware import HARDWARE
+from openpilot.system.hardware import HARDWARE, PC
+from openpilot.common.realtime import Ratekeeper
 
 DEFAULT_FPS = int(os.getenv("FPS", "60"))
 FPS_LOG_INTERVAL = 5  # Seconds between logging FPS drops
 FPS_DROP_THRESHOLD = 0.9  # FPS drop threshold for triggering a warning
 FPS_CRITICAL_THRESHOLD = 0.5  # Critical threshold for triggering strict actions
+MOUSE_THREAD_RATE = 140  # touch controller runs at 140Hz
 
 ENABLE_VSYNC = os.getenv("ENABLE_VSYNC", "0") == "1"
 SHOW_FPS = os.getenv("SHOW_FPS") == "1"
@@ -47,6 +51,68 @@ class ModalOverlay:
   callback: Callable | None = None
 
 
+class MousePos(NamedTuple):
+  x: float
+  y: float
+
+
+class MouseEvent(NamedTuple):
+  pos: MousePos
+  left_pressed: bool
+  left_released: bool
+  left_down: bool
+  t: float
+
+
+class MouseState:
+  def __init__(self):
+    self._events: deque[MouseEvent] = deque(maxlen=MOUSE_THREAD_RATE)  # bound event list
+    self._prev_mouse_event: MouseEvent | None = None
+
+    self._rk = Ratekeeper(MOUSE_THREAD_RATE)
+    self._lock = threading.Lock()
+    self._exit_event = threading.Event()
+    self._thread = None
+
+  def get_events(self) -> list[MouseEvent]:
+    with self._lock:
+      events = list(self._events)
+      self._events.clear()
+    return events
+
+  def start(self):
+    self._exit_event.clear()
+    if self._thread is None or not self._thread.is_alive():
+      self._thread = threading.Thread(target=self._run_thread, daemon=True)
+      self._thread.start()
+
+  def stop(self):
+    self._exit_event.set()
+    if self._thread is not None and self._thread.is_alive():
+      self._thread.join()
+
+  def _run_thread(self):
+    while not self._exit_event.is_set():
+      rl.poll_input_events()
+      self._handle_mouse_event()
+      self._rk.keep_time()
+
+  def _handle_mouse_event(self):
+    mouse_pos = rl.get_mouse_position()
+    ev = MouseEvent(
+      MousePos(mouse_pos.x, mouse_pos.y),
+      rl.is_mouse_button_pressed(rl.MouseButton.MOUSE_BUTTON_LEFT),
+      rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT),
+      rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT),
+      time.monotonic(),
+    )
+    # Only add changes
+    if self._prev_mouse_event is None or ev[:-1] != self._prev_mouse_event[:-1]:
+      with self._lock:
+        self._events.append(ev)
+      self._prev_mouse_event = ev
+
+
 class GuiApplication:
   def __init__(self, width: int, height: int):
     self._fonts: dict[FontWeight, rl.Font] = {}
@@ -63,8 +129,11 @@ class GuiApplication:
     self._trace_log_callback = None
     self._modal_overlay = ModalOverlay()
 
+    self._mouse = MouseState()
+    self._mouse_events: list[MouseEvent] = []
+
     # Debug variables
-    self._mouse_history: deque[rl.Vector2] = deque(maxlen=DEFAULT_FPS * 2)
+    self._mouse_history: deque[MousePos] = deque(maxlen=MOUSE_THREAD_RATE)
 
   def request_close(self):
     self._window_close_requested = True
@@ -94,6 +163,9 @@ class GuiApplication:
     self._set_styles()
     self._load_fonts()
 
+    if not PC:
+      self._mouse.start()
+
   def set_modal_overlay(self, overlay, callback: Callable | None = None):
     self._modal_overlay = ModalOverlay(overlay=overlay, callback=callback)
 
@@ -154,11 +226,25 @@ class GuiApplication:
       rl.unload_render_texture(self._render_texture)
       self._render_texture = None
 
+    if not PC:
+      self._mouse.stop()
+
     rl.close_window()
 
+  @property
+  def mouse_events(self) -> list[MouseEvent]:
+    return self._mouse_events
+
   def render(self):
     try:
       while not (self._window_close_requested or rl.window_should_close()):
+        if PC:
+          # Thread is not used on PC, need to manually add mouse events
+          self._mouse._handle_mouse_event()
+
+        # Store all mouse events for the current frame
+        self._mouse_events = self._mouse.get_events()
+
         if self._render_texture:
           rl.begin_texture_mode(self._render_texture)
           rl.clear_background(rl.BLACK)
@@ -196,9 +282,10 @@ class GuiApplication:
           rl.draw_fps(10, 10)
 
         if SHOW_TOUCHES:
-          if rl.is_mouse_button_pressed(rl.MouseButton.MOUSE_BUTTON_LEFT):
+          for mouse_event in self._mouse_events:
-            self._mouse_history.clear()
+            if mouse_event.left_pressed:
-          self._mouse_history.append(rl.get_mouse_position())
+              self._mouse_history.clear()
+            self._mouse_history.append(mouse_event.pos)
 
           if self._mouse_history:
             mouse_pos = self._mouse_history[-1]

system/ui/lib/list_view.py

@@ -2,7 +2,7 @@ import os
 import pyray as rl
 from collections.abc import Callable
 from abc import ABC
-from openpilot.system.ui.lib.application import gui_app, FontWeight
+from openpilot.system.ui.lib.application import gui_app, FontWeight, MousePos
 from openpilot.system.ui.lib.text_measure import measure_text_cached
 from openpilot.system.ui.lib.wrap_text import wrap_text
 from openpilot.system.ui.lib.button import gui_button, ButtonStyle
@@ -229,7 +229,7 @@ class ListItem(Widget):
     super().set_parent_rect(parent_rect)
     self._rect.width = parent_rect.width
 
-  def _handle_mouse_release(self, mouse_pos: rl.Vector2):
+  def _handle_mouse_release(self, mouse_pos: MousePos):
     if not self.is_visible:
       return
 

system/ui/lib/scroll_panel.py

@@ -1,6 +1,8 @@
+import time
 import pyray as rl
 from collections import deque
 from enum import IntEnum
+from openpilot.system.ui.lib.application import gui_app, MouseEvent, MousePos
 
 # Scroll constants for smooth scrolling behavior
 MOUSE_WHEEL_SCROLL_SPEED = 30
@@ -38,51 +40,54 @@ class GuiScrollPanel:
     self._bounds_rect: rl.Rectangle | None = None
 
   def handle_scroll(self, bounds: rl.Rectangle, content: rl.Rectangle) -> rl.Vector2:
+    # TODO: HACK: this class is driven by mouse events, so we need to ensure we have at least one event to process
+    for mouse_event in gui_app.mouse_events or [MouseEvent(MousePos(0, 0), False, False, False, time.monotonic())]:
+      self._handle_mouse_event(mouse_event, bounds, content)
+    return self._offset
+
+  def _handle_mouse_event(self, mouse_event: MouseEvent, bounds: rl.Rectangle, content: rl.Rectangle):
     # Store rectangles for reference
     self._content_rect = content
     self._bounds_rect = bounds
 
-    # Calculate time delta
-    current_time = rl.get_time()
-
-    mouse_pos = rl.get_mouse_position()
     max_scroll_y = max(content.height - bounds.height, 0)
 
     # Start dragging on mouse press
-    if rl.check_collision_point_rec(mouse_pos, bounds) and rl.is_mouse_button_pressed(rl.MouseButton.MOUSE_BUTTON_LEFT):
+    if rl.check_collision_point_rec(mouse_event.pos, bounds) and mouse_event.left_pressed:
       if self._scroll_state == ScrollState.IDLE or self._scroll_state == ScrollState.BOUNCING:
         self._scroll_state = ScrollState.DRAGGING_CONTENT
         if self._show_vertical_scroll_bar:
           scrollbar_width = rl.gui_get_style(rl.GuiControl.LISTVIEW, rl.GuiListViewProperty.SCROLLBAR_WIDTH)
           scrollbar_x = bounds.x + bounds.width - scrollbar_width
-          if mouse_pos.x >= scrollbar_x:
+          if mouse_event.pos.x >= scrollbar_x:
             self._scroll_state = ScrollState.DRAGGING_SCROLLBAR
 
         # TODO: hacky
         # when clicking while moving, go straight into dragging
         self._is_dragging = abs(self._velocity_y) > MIN_VELOCITY
-        self._last_mouse_y = mouse_pos.y
+        self._last_mouse_y = mouse_event.pos.y
-        self._start_mouse_y = mouse_pos.y
+        self._start_mouse_y = mouse_event.pos.y
-        self._last_drag_time = current_time
+        self._last_drag_time = mouse_event.t
         self._velocity_history.clear()
         self._velocity_y = 0.0
         self._bounce_offset = 0.0
 
     # Handle active dragging
     if self._scroll_state == ScrollState.DRAGGING_CONTENT or self._scroll_state == ScrollState.DRAGGING_SCROLLBAR:
-      if rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT):
+      if mouse_event.left_down:
-        delta_y = mouse_pos.y - self._last_mouse_y
+        delta_y = mouse_event.pos.y - self._last_mouse_y
 
         # Track velocity for inertia
-        time_since_last_drag = current_time - self._last_drag_time
+        time_since_last_drag = mouse_event.t - self._last_drag_time
         if time_since_last_drag > 0:
-          drag_velocity = delta_y / time_since_last_drag / 60.0
+          # TODO: HACK: /2 since we usually get two touch events per frame
+          drag_velocity = delta_y / time_since_last_drag / 60.0 / 2  # TODO: shouldn't be hardcoded
           self._velocity_history.append(drag_velocity)
 
-        self._last_drag_time = current_time
+        self._last_drag_time = mouse_event.t
 
         # Detect actual dragging
-        total_drag = abs(mouse_pos.y - self._start_mouse_y)
+        total_drag = abs(mouse_event.pos.y - self._start_mouse_y)
         if total_drag > DRAG_THRESHOLD:
           self._is_dragging = True
 
@@ -96,9 +101,9 @@ class GuiScrollPanel:
           scroll_ratio = content.height / bounds.height
           self._offset.y -= delta_y * scroll_ratio
 
-        self._last_mouse_y = mouse_pos.y
+        self._last_mouse_y = mouse_event.pos.y
 
-      elif rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT):
+      elif mouse_event.left_released:
         # Calculate flick velocity
         if self._velocity_history:
           total_weight = 0

system/ui/lib/toggle.py

@@ -1,4 +1,5 @@
 import pyray as rl
+from openpilot.system.ui.lib.application import MousePos
 from openpilot.system.ui.lib.widget import Widget
 
 ON_COLOR = rl.Color(51, 171, 76, 255)
@@ -23,7 +24,7 @@ class Toggle(Widget):
   def set_rect(self, rect: rl.Rectangle):
     self._rect = rl.Rectangle(rect.x, rect.y, WIDTH, HEIGHT)
 
-  def _handle_mouse_release(self, mouse_pos: rl.Vector2):
+  def _handle_mouse_release(self, mouse_pos: MousePos):
     if not self._enabled:
       return
 

system/ui/lib/widget.py

@@ -2,6 +2,7 @@ import abc
 import pyray as rl
 from enum import IntEnum
 from collections.abc import Callable
+from openpilot.system.ui.lib.application import gui_app, MousePos
 
 
 class DialogResult(IntEnum):
@@ -66,18 +67,18 @@ class Widget(abc.ABC):
     ret = self._render(self._rect)
 
     # Keep track of whether mouse down started within the widget's rectangle
-    mouse_pos = rl.get_mouse_position()
+    for mouse_event in gui_app.mouse_events:
-    if rl.is_mouse_button_pressed(rl.MouseButton.MOUSE_BUTTON_LEFT) and self._touch_valid():
+      if mouse_event.left_pressed and self._touch_valid():
-      if rl.check_collision_point_rec(mouse_pos, self._rect):
+        if rl.check_collision_point_rec(mouse_event.pos, self._rect):
-        self._is_pressed = True
+          self._is_pressed = True
 
-    elif not self._touch_valid():
+      elif not self._touch_valid():
-      self._is_pressed = False
+        self._is_pressed = False
 
-    elif rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT):
+      elif mouse_event.left_released:
-      if self._is_pressed and rl.check_collision_point_rec(mouse_pos, self._rect):
+        if self._is_pressed and rl.check_collision_point_rec(mouse_event.pos, self._rect):
-        self._handle_mouse_release(mouse_pos)
+          self._handle_mouse_release(mouse_event.pos)
-      self._is_pressed = False
+        self._is_pressed = False
 
     return ret
 
@@ -91,6 +92,6 @@ class Widget(abc.ABC):
   def _update_layout_rects(self) -> None:
     """Optionally update any layout rects on Widget rect change."""
 
-  def _handle_mouse_release(self, mouse_pos: rl.Vector2) -> bool:
+  def _handle_mouse_release(self, mouse_pos: MousePos) -> bool:
     """Optionally handle mouse release events."""
     return False