system/ui: add EGL support to CameraView (#35338)

* add EGL support to CameraView

* view 3 cameras

* use a more direct approach

* add new line

* cleanup

* cleanup close()

* extract EGL to a seperate file

* cleanup

* add try/except to close()

* rename egl_textures

* improve implementation

system/ui/lib/egl.py

@@ -0,0 +1,177 @@
+import os
+import cffi
+from dataclasses import dataclass
+from typing import Any
+from openpilot.common.swaglog import cloudlog
+
+
+# EGL constants
+EGL_LINUX_DMA_BUF_EXT = 0x3270
+EGL_WIDTH = 0x3057
+EGL_HEIGHT = 0x3056
+EGL_LINUX_DRM_FOURCC_EXT = 0x3271
+EGL_DMA_BUF_PLANE0_FD_EXT = 0x3272
+EGL_DMA_BUF_PLANE0_OFFSET_EXT = 0x3273
+EGL_DMA_BUF_PLANE0_PITCH_EXT = 0x3274
+EGL_DMA_BUF_PLANE1_FD_EXT = 0x3275
+EGL_DMA_BUF_PLANE1_OFFSET_EXT = 0x3276
+EGL_DMA_BUF_PLANE1_PITCH_EXT = 0x3277
+EGL_NONE = 0x3038
+GL_TEXTURE0 = 0x84C0
+GL_TEXTURE_EXTERNAL_OES = 0x8D65
+
+# DRM Format for NV12
+DRM_FORMAT_NV12 = 842094158
+
+@dataclass
+class EGLImage:
+  """Container for EGL image and associated resources"""
+
+  egl_image: Any
+  fd: int
+
+
+@dataclass
+class EGLState:
+  """Container for all EGL-related state"""
+
+  initialized: bool = False
+  ffi: Any = None
+  egl_lib: Any = None
+  gles_lib: Any = None
+
+  # EGL display connection - shared across all users
+  display: Any = None
+
+  # Constants
+  NO_CONTEXT: Any = None
+  NO_DISPLAY: Any = None
+  NO_IMAGE_KHR: Any = None
+
+  # Function pointers
+  get_current_display: Any = None
+  create_image_khr: Any = None
+  destroy_image_khr: Any = None
+  image_target_texture: Any = None
+  get_error: Any = None
+  bind_texture: Any = None
+  active_texture: Any = None
+
+
+# Create a single instance of the state
+_egl = EGLState()
+
+
+def init_egl() -> bool:
+  """Initialize EGL and load necessary functions"""
+  global _egl
+
+  # Don't re-initialize if already done
+  if _egl.initialized:
+    return True
+
+  try:
+    _egl.ffi = cffi.FFI()
+    _egl.ffi.cdef("""
+      typedef int EGLint;
+      typedef unsigned int EGLBoolean;
+      typedef unsigned int EGLenum;
+      typedef unsigned int GLenum;
+      typedef void *EGLContext;
+      typedef void *EGLDisplay;
+      typedef void *EGLClientBuffer;
+      typedef void *EGLImageKHR;
+      typedef void *GLeglImageOES;
+
+      EGLDisplay eglGetCurrentDisplay(void);
+      EGLint eglGetError(void);
+      EGLImageKHR eglCreateImageKHR(EGLDisplay dpy, EGLContext ctx,
+                                  EGLenum target, EGLClientBuffer buffer,
+                                  const EGLint *attrib_list);
+      EGLBoolean eglDestroyImageKHR(EGLDisplay dpy, EGLImageKHR image);
+      void glEGLImageTargetTexture2DOES(GLenum target, GLeglImageOES image);
+      void glBindTexture(GLenum target, unsigned int texture);
+      void glActiveTexture(GLenum texture);
+    """)
+
+    # Load libraries
+    _egl.egl_lib = _egl.ffi.dlopen("libEGL.so")
+    _egl.gles_lib = _egl.ffi.dlopen("libGLESv2.so")
+
+    # Cast NULL pointers
+    _egl.NO_CONTEXT = _egl.ffi.cast("void *", 0)
+    _egl.NO_DISPLAY = _egl.ffi.cast("void *", 0)
+    _egl.NO_IMAGE_KHR = _egl.ffi.cast("void *", 0)
+
+    # Bind functions
+    _egl.get_current_display = _egl.egl_lib.eglGetCurrentDisplay
+    _egl.create_image_khr = _egl.egl_lib.eglCreateImageKHR
+    _egl.destroy_image_khr = _egl.egl_lib.eglDestroyImageKHR
+    _egl.image_target_texture = _egl.gles_lib.glEGLImageTargetTexture2DOES
+    _egl.get_error = _egl.egl_lib.eglGetError
+    _egl.bind_texture = _egl.gles_lib.glBindTexture
+    _egl.active_texture = _egl.gles_lib.glActiveTexture
+
+    # Initialize EGL display once here
+    _egl.display = _egl.get_current_display()
+    if _egl.display == _egl.NO_DISPLAY:
+      raise RuntimeError("Failed to get EGL display")
+
+    _egl.initialized = True
+    return True
+  except Exception as e:
+    cloudlog.exception(f"EGL initialization failed: {e}")
+    _egl.initialized = False
+    return False
+
+
+def create_egl_image(width: int, height: int, stride: int, fd: int, uv_offset: int) -> EGLImage | None:
+  assert _egl.initialized, "EGL not initialized"
+
+  # Duplicate fd since EGL needs it
+  dup_fd = os.dup(fd)
+
+  # Create image attributes for EGL
+  img_attrs = [
+    EGL_WIDTH, width,
+    EGL_HEIGHT, height,
+    EGL_LINUX_DRM_FOURCC_EXT, DRM_FORMAT_NV12,
+    EGL_DMA_BUF_PLANE0_FD_EXT, dup_fd,
+    EGL_DMA_BUF_PLANE0_OFFSET_EXT, 0,
+    EGL_DMA_BUF_PLANE0_PITCH_EXT, stride,
+    EGL_DMA_BUF_PLANE1_FD_EXT, dup_fd,
+    EGL_DMA_BUF_PLANE1_OFFSET_EXT, uv_offset,
+    EGL_DMA_BUF_PLANE1_PITCH_EXT, stride,
+    EGL_NONE
+  ]
+
+  attr_array = _egl.ffi.new("int[]", img_attrs)
+  egl_image = _egl.create_image_khr(_egl.display, _egl.NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, _egl.ffi.NULL, attr_array)
+
+  if egl_image == _egl.NO_IMAGE_KHR:
+    cloudlog.error(f"Failed to create EGL image: {_egl.get_error()}")
+    os.close(dup_fd)
+    return None
+
+  return EGLImage(egl_image=egl_image, fd=dup_fd)
+
+
+def destroy_egl_image(egl_image: EGLImage) -> None:
+  assert _egl.initialized, "EGL not initialized"
+
+  _egl.destroy_image_khr(_egl.display, egl_image.egl_image)
+
+  # Close the duplicated fd we created in create_egl_image()
+  # We need to handle OSError since the fd might already be closed
+  try:
+    os.close(egl_image.fd)
+  except OSError:
+    pass
+
+
+def bind_egl_image_to_texture(texture_id: int, egl_image: EGLImage) -> None:
+  assert _egl.initialized, "EGL not initialized"
+
+  _egl.active_texture(GL_TEXTURE0)
+  _egl.bind_texture(GL_TEXTURE_EXTERNAL_OES, texture_id)
+  _egl.image_target_texture(GL_TEXTURE_EXTERNAL_OES, egl_image.egl_image)

system/ui/widgets/cameraview.py

@@ -1,6 +1,8 @@
 import pyray as rl
-from msgq.visionipc import VisionIpcClient, VisionStreamType
+from openpilot.system.hardware import TICI
+from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf
 from openpilot.system.ui.lib.application import gui_app
+from openpilot.system.ui.lib.egl import init_egl, create_egl_image, destroy_egl_image, bind_egl_image_to_texture, EGLImage
 
 
 VERTEX_SHADER = """
@@ -19,30 +21,67 @@ void main() {
 }
 """
 
-FRAME_FRAGMENT_SHADER = """
+# Choose fragment shader based on platform capabilities
-#version 300 es
+if TICI:
-precision mediump float;
+  FRAME_FRAGMENT_SHADER = """
-in vec2 fragTexCoord;
+    #version 300 es
-uniform sampler2D texture0;
+    #extension GL_OES_EGL_image_external_essl3 : enable
-uniform sampler2D texture1;
+    precision mediump float;
-out vec4 fragColor;
+    in vec2 fragTexCoord;
-void main() {
+    uniform samplerExternalOES texture0;
-  float y = texture(texture0, fragTexCoord).r;
+    out vec4 fragColor;
-  vec2 uv = texture(texture1, fragTexCoord).ra - 0.5;
+    void main() {
-  fragColor = vec4(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x, 1.0);
+      vec4 color = texture(texture0, fragTexCoord);
-}
+      fragColor = vec4(pow(color.rgb, vec3(1.0/1.28)), color.a);
-"""
+    }
+    """
+else:
+  FRAME_FRAGMENT_SHADER = """
+    #version 300 es
+    precision mediump float;
+    in vec2 fragTexCoord;
+    uniform sampler2D texture0;
+    uniform sampler2D texture1;
+    out vec4 fragColor;
+    void main() {
+      float y = texture(texture0, fragTexCoord).r;
+      vec2 uv = texture(texture1, fragTexCoord).ra - 0.5;
+      fragColor = vec4(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x, 1.0);
+    }
+    """
 
 class CameraView:
   def __init__(self, name: str, stream_type: VisionStreamType):
     self.client = VisionIpcClient(name, stream_type, False)
     self.shader = rl.load_shader_from_memory(VERTEX_SHADER, FRAME_FRAGMENT_SHADER)
+
+    self.frame: VisionBuf | None = None
     self.texture_y: rl.Texture | None = None
     self.texture_uv: rl.Texture | None = None
-    self.frame = None
 
-  def close(self):
+    # EGL resources
+    self.egl_images: dict[int, EGLImage] = {}
+    self.egl_texture: rl.Texture | None = None
+
+    # Initialize EGL for zero-copy rendering on TICI
+    if TICI:
+      if not init_egl():
+        raise RuntimeError("Failed to initialize EGL")
+
+      # Create a 1x1 pixel placeholder texture for EGL image binding
+      temp_image = rl.gen_image_color(1, 1, rl.BLACK)
+      self.egl_texture = rl.load_texture_from_image(temp_image)
+      rl.unload_image(temp_image)
+
+  def close(self) -> None:
     self._clear_textures()
+
+    # Clean up EGL texture
+    if TICI and self.egl_texture:
+      rl.unload_texture(self.egl_texture)
+      self.egl_texture = None
+
+    # Clean up shader
     if self.shader and self.shader.id:
       rl.unload_shader(self.shader)
 
@@ -50,16 +89,13 @@ class CameraView:
     if not self._ensure_connection():
       return
 
+    # Try to get a new buffer without blocking
     buffer = self.client.recv(timeout_ms=0)
-    self.frame = buffer if buffer else self.frame
+    if buffer:
-    if not self.frame or not self.texture_y or not self.texture_uv:
+      self.frame = buffer
-      return
 
-    y_data = self.frame.data[: self.frame.uv_offset]
+    if not self.frame:
-    uv_data = self.frame.data[self.frame.uv_offset :]
+      return
-
-    rl.update_texture(self.texture_y, rl.ffi.cast("void *", y_data.ctypes.data))
-    rl.update_texture(self.texture_uv, rl.ffi.cast("void *", uv_data.ctypes.data))
 
     # Calculate scaling to maintain aspect ratio
     scale = min(rect.width / self.frame.width, rect.height / self.frame.height)
@@ -68,6 +104,53 @@ class CameraView:
     src_rect = rl.Rectangle(0, 0, float(self.frame.width), float(self.frame.height))
     dst_rect = rl.Rectangle(x_offset, y_offset, self.frame.width * scale, self.frame.height * scale)
 
+    # Render with appropriate method
+    if TICI:
+      self._render_egl(src_rect, dst_rect)
+    else:
+      self._render_textures(src_rect, dst_rect)
+
+  def _render_egl(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None:
+    """Render using EGL for direct buffer access"""
+    if self.frame is None or self.egl_texture is None:
+      return
+
+    idx = self.frame.idx
+    egl_image = self.egl_images.get(idx)
+
+    # Create EGL image if needed
+    if egl_image is None:
+      egl_image = create_egl_image(self.frame.width, self.frame.height, self.frame.stride, self.frame.fd, self.frame.uv_offset)
+      if egl_image:
+        self.egl_images[idx] = egl_image
+      else:
+        return
+
+    # Update texture dimensions to match current frame
+    self.egl_texture.width = self.frame.width
+    self.egl_texture.height = self.frame.height
+
+    # Bind the EGL image to our texture
+    bind_egl_image_to_texture(self.egl_texture.id, egl_image)
+
+    # Render with shader
+    rl.begin_shader_mode(self.shader)
+    rl.draw_texture_pro(self.egl_texture, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
+    rl.end_shader_mode()
+
+  def _render_textures(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None:
+    """Render using texture copies"""
+    if not self.texture_y or not self.texture_uv or self.frame is None:
+      return
+
+    # Update textures with new frame data
+    y_data = self.frame.data[: self.frame.uv_offset]
+    uv_data = self.frame.data[self.frame.uv_offset :]
+
+    rl.update_texture(self.texture_y, rl.ffi.cast("void *", y_data.ctypes.data))
+    rl.update_texture(self.texture_uv, rl.ffi.cast("void *", uv_data.ctypes.data))
+
+    # Render with shader
     rl.begin_shader_mode(self.shader)
     rl.set_shader_value_texture(self.shader, rl.get_shader_location(self.shader, "texture1"), self.texture_uv)
     rl.draw_texture_pro(self.texture_y, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
@@ -80,17 +163,30 @@ class CameraView:
         return False
 
       self._clear_textures()
-      self.texture_y = rl.load_texture_from_image(rl.Image(None, int(self.client.stride),
+
-        int(self.client.height), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAYSCALE))
+      if not TICI:
-      self.texture_uv = rl.load_texture_from_image(rl.Image(None, int(self.client.stride // 2),
+        self.texture_y = rl.load_texture_from_image(rl.Image(None, int(self.client.stride),
-        int(self.client.height // 2), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA))
+          int(self.client.height), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAYSCALE))
+        self.texture_uv = rl.load_texture_from_image(rl.Image(None, int(self.client.stride // 2),
+          int(self.client.height // 2), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA))
+
     return True
 
   def _clear_textures(self):
     if self.texture_y and self.texture_y.id:
       rl.unload_texture(self.texture_y)
+      self.texture_y = None
+
     if self.texture_uv and self.texture_uv.id:
       rl.unload_texture(self.texture_uv)
+      self.texture_uv = None
+
+    # Clean up EGL resources
+    if TICI:
+      for data in self.egl_images.values():
+        destroy_egl_image(data)
+      self.egl_images = {}
+
 
 if __name__ == "__main__":
   gui_app.init_window("watch3")