diff --git a/selfdrive/ui/layouts/sidebar.py b/selfdrive/ui/layouts/sidebar.py
index 82d499716d..6af947a104 100644
--- a/selfdrive/ui/layouts/sidebar.py
+++ b/selfdrive/ui/layouts/sidebar.py
@@ -146,7 +146,7 @@ class Sidebar(Widget):
 
   def _draw_buttons(self, rect: rl.Rectangle):
     mouse_pos = rl.get_mouse_position()
-    mouse_down = self._is_pressed and rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT)
+    mouse_down = self.is_pressed and rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT)
 
     # Settings button
     settings_down = mouse_down and rl.check_collision_point_rec(mouse_pos, SETTINGS_BTN)
diff --git a/selfdrive/ui/onroad/exp_button.py b/selfdrive/ui/onroad/exp_button.py
index e6748e4e09..27f5763077 100644
--- a/selfdrive/ui/onroad/exp_button.py
+++ b/selfdrive/ui/onroad/exp_button.py
@@ -50,7 +50,7 @@ class ExpButton(Widget):
     center_y = int(self._rect.y + self._rect.height // 2)
 
     mouse_over = rl.check_collision_point_rec(rl.get_mouse_position(), self._rect)
-    mouse_down = rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT) and self._is_pressed
+    mouse_down = rl.is_mouse_button_down(rl.MouseButton.MOUSE_BUTTON_LEFT) and self.is_pressed
     self._white_color.a = 180 if (mouse_down and mouse_over) or not self._engageable else 255
 
     texture = self._txt_exp if self._held_or_actual_mode() else self._txt_wheel
diff --git a/selfdrive/ui/widgets/exp_mode_button.py b/selfdrive/ui/widgets/exp_mode_button.py
index be50c59d06..88664e0fad 100644
--- a/selfdrive/ui/widgets/exp_mode_button.py
+++ b/selfdrive/ui/widgets/exp_mode_button.py
@@ -14,7 +14,6 @@ class ExperimentalModeButton(Widget):
 
     self.params = Params()
     self.experimental_mode = self.params.get_bool("ExperimentalMode")
-    self.is_pressed = False
 
     self.chill_pixmap = gui_app.texture("icons/couch.png", self.img_width, self.img_width)
     self.experimental_pixmap = gui_app.texture("icons/experimental_grey.png", self.img_width, self.img_width)
@@ -32,19 +31,12 @@ class ExperimentalModeButton(Widget):
     rl.draw_rectangle_gradient_h(int(rect.x), int(rect.y), int(rect.width), int(rect.height),
                                  start_color, end_color)
 
-  def _handle_interaction(self, rect):
-    mouse_pos = rl.get_mouse_position()
-    mouse_in_rect = rl.check_collision_point_rec(mouse_pos, rect)
-
-    self.is_pressed = mouse_in_rect and rl.is_mouse_button_down(rl.MOUSE_BUTTON_LEFT)
-    return mouse_in_rect and rl.is_mouse_button_released(rl.MOUSE_BUTTON_LEFT)
+  def _handle_mouse_release(self, mouse_pos):
+    self.experimental_mode = not self.experimental_mode
+    # TODO: Opening settings for ExperimentalMode
+    self.params.put_bool("ExperimentalMode", self.experimental_mode)
 
   def _render(self, rect):
-    if self._handle_interaction(rect):
-      self.experimental_mode = not self.experimental_mode
-      # TODO: Opening settings for ExperimentalMode
-      self.params.put_bool("ExperimentalMode", self.experimental_mode)
-
     rl.draw_rectangle_rounded(rect, 0.08, 20, rl.Color(255, 255, 255, 255))
 
     rl.begin_scissor_mode(int(rect.x), int(rect.y), int(rect.width), int(rect.height))
diff --git a/system/ui/widgets/__init__.py b/system/ui/widgets/__init__.py
index a2c34799f5..dc87216031 100644
--- a/system/ui/widgets/__init__.py
+++ b/system/ui/widgets/__init__.py
@@ -16,6 +16,8 @@ class Widget(abc.ABC):
     self._rect: rl.Rectangle = rl.Rectangle(0, 0, 0, 0)
     self._parent_rect: rl.Rectangle = rl.Rectangle(0, 0, 0, 0)
     self._is_pressed = [False] * MAX_TOUCH_SLOTS
+    # if current mouse/touch down started within the widget's rectangle
+    self._tracking_is_pressed = [False] * MAX_TOUCH_SLOTS
     self._enabled: bool | Callable[[], bool] = True
     self._is_visible: bool | Callable[[], bool] = True
     self._touch_valid_callback: Callable[[], bool] | None = None
@@ -83,17 +85,33 @@ class Widget(abc.ABC):
     for mouse_event in gui_app.mouse_events:
       if not self._multi_touch and mouse_event.slot != 0:
         continue
+
+      # Ignores touches/presses that start outside our rect
+      # Allows touch to leave the rect and come back in focus if mouse did not release
       if mouse_event.left_pressed and self._touch_valid():
         if rl.check_collision_point_rec(mouse_event.pos, self._rect):
           self._is_pressed[mouse_event.slot] = True
+          self._tracking_is_pressed[mouse_event.slot] = True
 
+      # Callback such as scroll panel signifies user is scrolling
       elif not self._touch_valid():
         self._is_pressed[mouse_event.slot] = False
+        self._tracking_is_pressed[mouse_event.slot] = False
 
       elif mouse_event.left_released:
         if self._is_pressed[mouse_event.slot] and rl.check_collision_point_rec(mouse_event.pos, self._rect):
           self._handle_mouse_release(mouse_event.pos)
         self._is_pressed[mouse_event.slot] = False
+        self._tracking_is_pressed[mouse_event.slot] = False
+
+      # Mouse/touch is still within our rect
+      elif rl.check_collision_point_rec(mouse_event.pos, self._rect):
+        if self._tracking_is_pressed[mouse_event.slot]:
+          self._is_pressed[mouse_event.slot] = True
+
+      # Mouse/touch left our rect but may come back into focus later
+      elif not rl.check_collision_point_rec(mouse_event.pos, self._rect):
+        self._is_pressed[mouse_event.slot] = False
 
     return ret