ui: multi touch keyboard support (#35912)

* start

* better

* 2

* dumb

system/ui/lib/application.py

@@ -19,6 +19,7 @@ FPS_LOG_INTERVAL = 5  # Seconds between logging FPS drops
 FPS_DROP_THRESHOLD = 0.9  # FPS drop threshold for triggering a warning
 FPS_CRITICAL_THRESHOLD = 0.5  # Critical threshold for triggering strict actions
 MOUSE_THREAD_RATE = 140  # touch controller runs at 140Hz
+MAX_TOUCH_SLOT = 2
 
 ENABLE_VSYNC = os.getenv("ENABLE_VSYNC", "0") == "1"
 SHOW_FPS = os.getenv("SHOW_FPS") == "1"
@@ -58,6 +59,7 @@ class MousePos(NamedTuple):
 
 class MouseEvent(NamedTuple):
   pos: MousePos
+  slot: int
   left_pressed: bool
   left_released: bool
   left_down: bool
@@ -67,7 +69,7 @@ class MouseEvent(NamedTuple):
 class MouseState:
   def __init__(self):
     self._events: deque[MouseEvent] = deque(maxlen=MOUSE_THREAD_RATE)  # bound event list
-    self._prev_mouse_event: MouseEvent | None = None
+    self._prev_mouse_event: list[MouseEvent | None] = [None] * MAX_TOUCH_SLOT
 
     self._rk = Ratekeeper(MOUSE_THREAD_RATE)
     self._lock = threading.Lock()
@@ -98,19 +100,21 @@ class MouseState:
       self._rk.keep_time()
 
   def _handle_mouse_event(self):
-    mouse_pos = rl.get_mouse_position()
-    ev = MouseEvent(
-      MousePos(mouse_pos.x, mouse_pos.y),
-      rl.is_mouse_button_pressed(rl.MouseButton.MOUSE_BUTTON_LEFT),
-      rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT),
-      rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT),
-      time.monotonic(),
-    )
-    # Only add changes
-    if self._prev_mouse_event is None or ev[:-1] != self._prev_mouse_event[:-1]:
-      with self._lock:
-        self._events.append(ev)
-      self._prev_mouse_event = ev
+    for slot in range(MAX_TOUCH_SLOT):
+      mouse_pos = rl.get_touch_position(slot)
+      ev = MouseEvent(
+        MousePos(mouse_pos.x, mouse_pos.y),
+        slot,
+        rl.is_mouse_button_pressed(slot),
+        rl.is_mouse_button_released(slot),
+        rl.is_mouse_button_down(slot),
+        time.monotonic(),
+      )
+      # Only add changes
+      if self._prev_mouse_event[slot] is None or ev[:-1] != self._prev_mouse_event[slot][:-1]:
+        with self._lock:
+          self._events.append(ev)
+        self._prev_mouse_event[slot] = ev
 
 
 class GuiApplication:

system/ui/lib/scroll_panel.py

@@ -41,8 +41,9 @@ class GuiScrollPanel:
 
   def handle_scroll(self, bounds: rl.Rectangle, content: rl.Rectangle) -> rl.Vector2:
     # TODO: HACK: this class is driven by mouse events, so we need to ensure we have at least one event to process
-    for mouse_event in gui_app.mouse_events or [MouseEvent(MousePos(0, 0), False, False, False, time.monotonic())]:
-      self._handle_mouse_event(mouse_event, bounds, content)
+    for mouse_event in gui_app.mouse_events or [MouseEvent(MousePos(0, 0), 0, False, False, False, time.monotonic())]:
+      if mouse_event.slot == 0:
+        self._handle_mouse_event(mouse_event, bounds, content)
     return self._offset
 
   def _handle_mouse_event(self, mouse_event: MouseEvent, bounds: rl.Rectangle, content: rl.Rectangle):

system/ui/widgets/__init__.py

@@ -2,7 +2,7 @@ import abc
 import pyray as rl
 from enum import IntEnum
 from collections.abc import Callable
-from openpilot.system.ui.lib.application import gui_app, MousePos
+from openpilot.system.ui.lib.application import gui_app, MousePos, MAX_TOUCH_SLOT
 
 
 class DialogResult(IntEnum):
@@ -15,7 +15,8 @@ class Widget(abc.ABC):
   def __init__(self):
     self._rect: rl.Rectangle = rl.Rectangle(0, 0, 0, 0)
     self._parent_rect: rl.Rectangle = rl.Rectangle(0, 0, 0, 0)
-    self._is_pressed = False
+    self._is_pressed = [False] * MAX_TOUCH_SLOT
+    self._multi_touch = False
     self._is_visible: bool | Callable[[], bool] = True
     self._touch_valid_callback: Callable[[], bool] | None = None
 
@@ -31,6 +32,10 @@ class Widget(abc.ABC):
   def is_visible(self) -> bool:
     return self._is_visible() if callable(self._is_visible) else self._is_visible
 
+  @property
+  def is_pressed(self) -> bool:
+    return any(self._is_pressed)
+
   @property
   def rect(self) -> rl.Rectangle:
     return self._rect
@@ -68,17 +73,19 @@ class Widget(abc.ABC):
 
     # Keep track of whether mouse down started within the widget's rectangle
     for mouse_event in gui_app.mouse_events:
+      if not self._multi_touch and mouse_event.slot != 0:
+        continue
       if mouse_event.left_pressed and self._touch_valid():
         if rl.check_collision_point_rec(mouse_event.pos, self._rect):
-          self._is_pressed = True
+          self._is_pressed[mouse_event.slot] = True
 
       elif not self._touch_valid():
-        self._is_pressed = False
+        self._is_pressed[mouse_event.slot] = False
 
       elif mouse_event.left_released:
-        if self._is_pressed and rl.check_collision_point_rec(mouse_event.pos, self._rect):
+        if self._is_pressed[mouse_event.slot] and rl.check_collision_point_rec(mouse_event.pos, self._rect):
           self._handle_mouse_release(mouse_event.pos)
-        self._is_pressed = False
+        self._is_pressed[mouse_event.slot] = False
 
     return ret
 

system/ui/widgets/button.py

@@ -179,6 +179,7 @@ class Button(Widget):
                text_padding: int = 20,
                enabled: bool = True,
                icon = None,
+               multi_touch: bool = False,
                ):
 
     super().__init__()
@@ -195,6 +196,7 @@ class Button(Widget):
     self._text_padding = text_padding
     self._text_size = measure_text_cached(gui_app.font(self._font_weight), self._text, self._font_size)
     self._icon = icon
+    self._multi_touch = multi_touch
     self.enabled = enabled
 
   def set_text(self, text):
@@ -208,7 +210,7 @@ class Button(Widget):
   def _update_state(self):
     if self.enabled:
       self._text_color = BUTTON_TEXT_COLOR[self._button_style]
-      if self._is_pressed:
+      if self.is_pressed:
         self._background_color = BUTTON_PRESSED_BACKGROUND_COLORS[self._button_style]
       else:
         self._background_color = BUTTON_BACKGROUND_COLORS[self._button_style]

system/ui/widgets/keyboard.py

@@ -98,11 +98,11 @@ class Keyboard(Widget):
           if key in self._key_icons:
             texture = self._key_icons[key]
             self._all_keys[key] = Button("", partial(self._key_callback, key), icon=texture,
-                                        button_style=ButtonStyle.PRIMARY if key == ENTER_KEY else ButtonStyle.KEYBOARD)
+                                        button_style=ButtonStyle.PRIMARY if key == ENTER_KEY else ButtonStyle.KEYBOARD, multi_touch=True)
           else:
-            self._all_keys[key] = Button(key, partial(self._key_callback, key), button_style=ButtonStyle.KEYBOARD, font_size=85)
+            self._all_keys[key] = Button(key, partial(self._key_callback, key), button_style=ButtonStyle.KEYBOARD, font_size=85, multi_touch=True)
     self._all_keys[CAPS_LOCK_KEY] = Button("", partial(self._key_callback, CAPS_LOCK_KEY), icon=self._key_icons[CAPS_LOCK_KEY],
-                                           button_style=ButtonStyle.KEYBOARD)
+                                           button_style=ButtonStyle.KEYBOARD, multi_touch=True)
 
   @property
   def text(self):
@@ -143,7 +143,7 @@ class Keyboard(Widget):
     self._render_input_area(input_box_rect)
 
     # Process backspace key repeat if it's held down
-    if not self._all_keys[BACKSPACE_KEY]._is_pressed:
+    if not self._all_keys[BACKSPACE_KEY].is_pressed:
       self._backspace_pressed = False
 
     if self._backspace_pressed:
@@ -179,7 +179,7 @@ class Keyboard(Widget):
 
         is_enabled = key != ENTER_KEY or len(self._input_box.text) >= self._min_text_size
 
-        if key == BACKSPACE_KEY and self._all_keys[BACKSPACE_KEY]._is_pressed and not self._backspace_pressed:
+        if key == BACKSPACE_KEY and self._all_keys[BACKSPACE_KEY].is_pressed and not self._backspace_pressed:
           self._backspace_pressed = True
           self._backspace_press_time = time.monotonic()
           self._backspace_last_repeat = time.monotonic()

system/ui/widgets/list_view.py

@@ -167,7 +167,7 @@ class MultipleButtonAction(ItemAction):
       # Check button state
       mouse_pos = rl.get_mouse_position()
       is_hovered = rl.check_collision_point_rec(mouse_pos, button_rect)
-      is_pressed = is_hovered and rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT) and self._is_pressed
+      is_pressed = is_hovered and rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT) and self.is_pressed
       is_selected = i == self.selected_button
 
       # Button colors
@@ -188,7 +188,7 @@ class MultipleButtonAction(ItemAction):
       rl.draw_text_ex(self._font, text, rl.Vector2(text_x, text_y), 40, 0, rl.Color(228, 228, 228, 255))
 
       # Handle click
-      if is_hovered and rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT) and self._is_pressed:
+      if is_hovered and rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT) and self.is_pressed:
         clicked = i
 
     if clicked >= 0: