From 976dfa3982890ea69a981ef9988ee8e76f2a56f0 Mon Sep 17 00:00:00 2001 From: Maxime Desroches Date: Sun, 3 Aug 2025 18:14:48 -0700 Subject: [PATCH] ui: multi touch keyboard support (#35912) * start * better * 2 * dumb --- system/ui/lib/application.py | 32 ++++++++++++++++++-------------- system/ui/lib/scroll_panel.py | 5 +++-- system/ui/widgets/__init__.py | 19 +++++++++++++------ system/ui/widgets/button.py | 4 +++- system/ui/widgets/keyboard.py | 10 +++++----- system/ui/widgets/list_view.py | 4 ++-- 6 files changed, 44 insertions(+), 30 deletions(-) diff --git a/system/ui/lib/application.py b/system/ui/lib/application.py index c7fe39adf9..30672fba06 100644 --- a/system/ui/lib/application.py +++ b/system/ui/lib/application.py @@ -19,6 +19,7 @@ FPS_LOG_INTERVAL = 5 # Seconds between logging FPS drops FPS_DROP_THRESHOLD = 0.9 # FPS drop threshold for triggering a warning FPS_CRITICAL_THRESHOLD = 0.5 # Critical threshold for triggering strict actions MOUSE_THREAD_RATE = 140 # touch controller runs at 140Hz +MAX_TOUCH_SLOT = 2 ENABLE_VSYNC = os.getenv("ENABLE_VSYNC", "0") == "1" SHOW_FPS = os.getenv("SHOW_FPS") == "1" @@ -58,6 +59,7 @@ class MousePos(NamedTuple): class MouseEvent(NamedTuple): pos: MousePos + slot: int left_pressed: bool left_released: bool left_down: bool @@ -67,7 +69,7 @@ class MouseEvent(NamedTuple): class MouseState: def __init__(self): self._events: deque[MouseEvent] = deque(maxlen=MOUSE_THREAD_RATE) # bound event list - self._prev_mouse_event: MouseEvent | None = None + self._prev_mouse_event: list[MouseEvent | None] = [None] * MAX_TOUCH_SLOT self._rk = Ratekeeper(MOUSE_THREAD_RATE) self._lock = threading.Lock() @@ -98,19 +100,21 @@ class MouseState: self._rk.keep_time() def _handle_mouse_event(self): - mouse_pos = rl.get_mouse_position() - ev = MouseEvent( - MousePos(mouse_pos.x, mouse_pos.y), - rl.is_mouse_button_pressed(rl.MouseButton.MOUSE_BUTTON_LEFT), - rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT), - rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT), - time.monotonic(), - ) - # Only add changes - if self._prev_mouse_event is None or ev[:-1] != self._prev_mouse_event[:-1]: - with self._lock: - self._events.append(ev) - self._prev_mouse_event = ev + for slot in range(MAX_TOUCH_SLOT): + mouse_pos = rl.get_touch_position(slot) + ev = MouseEvent( + MousePos(mouse_pos.x, mouse_pos.y), + slot, + rl.is_mouse_button_pressed(slot), + rl.is_mouse_button_released(slot), + rl.is_mouse_button_down(slot), + time.monotonic(), + ) + # Only add changes + if self._prev_mouse_event[slot] is None or ev[:-1] != self._prev_mouse_event[slot][:-1]: + with self._lock: + self._events.append(ev) + self._prev_mouse_event[slot] = ev class GuiApplication: diff --git a/system/ui/lib/scroll_panel.py b/system/ui/lib/scroll_panel.py index 21b6795a75..e2296fd5ed 100644 --- a/system/ui/lib/scroll_panel.py +++ b/system/ui/lib/scroll_panel.py @@ -41,8 +41,9 @@ class GuiScrollPanel: def handle_scroll(self, bounds: rl.Rectangle, content: rl.Rectangle) -> rl.Vector2: # TODO: HACK: this class is driven by mouse events, so we need to ensure we have at least one event to process - for mouse_event in gui_app.mouse_events or [MouseEvent(MousePos(0, 0), False, False, False, time.monotonic())]: - self._handle_mouse_event(mouse_event, bounds, content) + for mouse_event in gui_app.mouse_events or [MouseEvent(MousePos(0, 0), 0, False, False, False, time.monotonic())]: + if mouse_event.slot == 0: + self._handle_mouse_event(mouse_event, bounds, content) return self._offset def _handle_mouse_event(self, mouse_event: MouseEvent, bounds: rl.Rectangle, content: rl.Rectangle): diff --git a/system/ui/widgets/__init__.py b/system/ui/widgets/__init__.py index 7a436b8e9f..2d619cbd11 100644 --- a/system/ui/widgets/__init__.py +++ b/system/ui/widgets/__init__.py @@ -2,7 +2,7 @@ import abc import pyray as rl from enum import IntEnum from collections.abc import Callable -from openpilot.system.ui.lib.application import gui_app, MousePos +from openpilot.system.ui.lib.application import gui_app, MousePos, MAX_TOUCH_SLOT class DialogResult(IntEnum): @@ -15,7 +15,8 @@ class Widget(abc.ABC): def __init__(self): self._rect: rl.Rectangle = rl.Rectangle(0, 0, 0, 0) self._parent_rect: rl.Rectangle = rl.Rectangle(0, 0, 0, 0) - self._is_pressed = False + self._is_pressed = [False] * MAX_TOUCH_SLOT + self._multi_touch = False self._is_visible: bool | Callable[[], bool] = True self._touch_valid_callback: Callable[[], bool] | None = None @@ -31,6 +32,10 @@ class Widget(abc.ABC): def is_visible(self) -> bool: return self._is_visible() if callable(self._is_visible) else self._is_visible + @property + def is_pressed(self) -> bool: + return any(self._is_pressed) + @property def rect(self) -> rl.Rectangle: return self._rect @@ -68,17 +73,19 @@ class Widget(abc.ABC): # Keep track of whether mouse down started within the widget's rectangle for mouse_event in gui_app.mouse_events: + if not self._multi_touch and mouse_event.slot != 0: + continue if mouse_event.left_pressed and self._touch_valid(): if rl.check_collision_point_rec(mouse_event.pos, self._rect): - self._is_pressed = True + self._is_pressed[mouse_event.slot] = True elif not self._touch_valid(): - self._is_pressed = False + self._is_pressed[mouse_event.slot] = False elif mouse_event.left_released: - if self._is_pressed and rl.check_collision_point_rec(mouse_event.pos, self._rect): + if self._is_pressed[mouse_event.slot] and rl.check_collision_point_rec(mouse_event.pos, self._rect): self._handle_mouse_release(mouse_event.pos) - self._is_pressed = False + self._is_pressed[mouse_event.slot] = False return ret diff --git a/system/ui/widgets/button.py b/system/ui/widgets/button.py index 8b7f52129c..04fed82b34 100644 --- a/system/ui/widgets/button.py +++ b/system/ui/widgets/button.py @@ -179,6 +179,7 @@ class Button(Widget): text_padding: int = 20, enabled: bool = True, icon = None, + multi_touch: bool = False, ): super().__init__() @@ -195,6 +196,7 @@ class Button(Widget): self._text_padding = text_padding self._text_size = measure_text_cached(gui_app.font(self._font_weight), self._text, self._font_size) self._icon = icon + self._multi_touch = multi_touch self.enabled = enabled def set_text(self, text): @@ -208,7 +210,7 @@ class Button(Widget): def _update_state(self): if self.enabled: self._text_color = BUTTON_TEXT_COLOR[self._button_style] - if self._is_pressed: + if self.is_pressed: self._background_color = BUTTON_PRESSED_BACKGROUND_COLORS[self._button_style] else: self._background_color = BUTTON_BACKGROUND_COLORS[self._button_style] diff --git a/system/ui/widgets/keyboard.py b/system/ui/widgets/keyboard.py index b34b4d6a4e..388d7e2664 100644 --- a/system/ui/widgets/keyboard.py +++ b/system/ui/widgets/keyboard.py @@ -98,11 +98,11 @@ class Keyboard(Widget): if key in self._key_icons: texture = self._key_icons[key] self._all_keys[key] = Button("", partial(self._key_callback, key), icon=texture, - button_style=ButtonStyle.PRIMARY if key == ENTER_KEY else ButtonStyle.KEYBOARD) + button_style=ButtonStyle.PRIMARY if key == ENTER_KEY else ButtonStyle.KEYBOARD, multi_touch=True) else: - self._all_keys[key] = Button(key, partial(self._key_callback, key), button_style=ButtonStyle.KEYBOARD, font_size=85) + self._all_keys[key] = Button(key, partial(self._key_callback, key), button_style=ButtonStyle.KEYBOARD, font_size=85, multi_touch=True) self._all_keys[CAPS_LOCK_KEY] = Button("", partial(self._key_callback, CAPS_LOCK_KEY), icon=self._key_icons[CAPS_LOCK_KEY], - button_style=ButtonStyle.KEYBOARD) + button_style=ButtonStyle.KEYBOARD, multi_touch=True) @property def text(self): @@ -143,7 +143,7 @@ class Keyboard(Widget): self._render_input_area(input_box_rect) # Process backspace key repeat if it's held down - if not self._all_keys[BACKSPACE_KEY]._is_pressed: + if not self._all_keys[BACKSPACE_KEY].is_pressed: self._backspace_pressed = False if self._backspace_pressed: @@ -179,7 +179,7 @@ class Keyboard(Widget): is_enabled = key != ENTER_KEY or len(self._input_box.text) >= self._min_text_size - if key == BACKSPACE_KEY and self._all_keys[BACKSPACE_KEY]._is_pressed and not self._backspace_pressed: + if key == BACKSPACE_KEY and self._all_keys[BACKSPACE_KEY].is_pressed and not self._backspace_pressed: self._backspace_pressed = True self._backspace_press_time = time.monotonic() self._backspace_last_repeat = time.monotonic() diff --git a/system/ui/widgets/list_view.py b/system/ui/widgets/list_view.py index aa2fdb845d..e871eef0a1 100644 --- a/system/ui/widgets/list_view.py +++ b/system/ui/widgets/list_view.py @@ -167,7 +167,7 @@ class MultipleButtonAction(ItemAction): # Check button state mouse_pos = rl.get_mouse_position() is_hovered = rl.check_collision_point_rec(mouse_pos, button_rect) - is_pressed = is_hovered and rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT) and self._is_pressed + is_pressed = is_hovered and rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT) and self.is_pressed is_selected = i == self.selected_button # Button colors @@ -188,7 +188,7 @@ class MultipleButtonAction(ItemAction): rl.draw_text_ex(self._font, text, rl.Vector2(text_x, text_y), 40, 0, rl.Color(228, 228, 228, 255)) # Handle click - if is_hovered and rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT) and self._is_pressed: + if is_hovered and rl.is_mouse_button_released(rl.MouseButton.MOUSE_BUTTON_LEFT) and self.is_pressed: clicked = i if clicked >= 0: