From 840ced5005eefacde09976555d0226155f849cb9 Mon Sep 17 00:00:00 2001 From: Dean Lee Date: Sun, 25 May 2025 23:41:03 +0800 Subject: [PATCH] system/ui: add EGL support to CameraView (#35338) * add EGL support to CameraView * view 3 cameras * use a more direct approach * add new line * cleanup * cleanup close() * extract EGL to a seperate file * cleanup * add try/except to close() * rename egl_textures * improve implementation --- system/ui/lib/egl.py | 177 ++++++++++++++++++++++++++++++++ system/ui/widgets/cameraview.py | 152 ++++++++++++++++++++++----- 2 files changed, 301 insertions(+), 28 deletions(-) create mode 100644 system/ui/lib/egl.py diff --git a/system/ui/lib/egl.py b/system/ui/lib/egl.py new file mode 100644 index 0000000000..d43be482b3 --- /dev/null +++ b/system/ui/lib/egl.py @@ -0,0 +1,177 @@ +import os +import cffi +from dataclasses import dataclass +from typing import Any +from openpilot.common.swaglog import cloudlog + + +# EGL constants +EGL_LINUX_DMA_BUF_EXT = 0x3270 +EGL_WIDTH = 0x3057 +EGL_HEIGHT = 0x3056 +EGL_LINUX_DRM_FOURCC_EXT = 0x3271 +EGL_DMA_BUF_PLANE0_FD_EXT = 0x3272 +EGL_DMA_BUF_PLANE0_OFFSET_EXT = 0x3273 +EGL_DMA_BUF_PLANE0_PITCH_EXT = 0x3274 +EGL_DMA_BUF_PLANE1_FD_EXT = 0x3275 +EGL_DMA_BUF_PLANE1_OFFSET_EXT = 0x3276 +EGL_DMA_BUF_PLANE1_PITCH_EXT = 0x3277 +EGL_NONE = 0x3038 +GL_TEXTURE0 = 0x84C0 +GL_TEXTURE_EXTERNAL_OES = 0x8D65 + +# DRM Format for NV12 +DRM_FORMAT_NV12 = 842094158 + +@dataclass +class EGLImage: + """Container for EGL image and associated resources""" + + egl_image: Any + fd: int + + +@dataclass +class EGLState: + """Container for all EGL-related state""" + + initialized: bool = False + ffi: Any = None + egl_lib: Any = None + gles_lib: Any = None + + # EGL display connection - shared across all users + display: Any = None + + # Constants + NO_CONTEXT: Any = None + NO_DISPLAY: Any = None + NO_IMAGE_KHR: Any = None + + # Function pointers + get_current_display: Any = None + create_image_khr: Any = None + destroy_image_khr: Any = None + image_target_texture: Any = None + get_error: Any = None + bind_texture: Any = None + active_texture: Any = None + + +# Create a single instance of the state +_egl = EGLState() + + +def init_egl() -> bool: + """Initialize EGL and load necessary functions""" + global _egl + + # Don't re-initialize if already done + if _egl.initialized: + return True + + try: + _egl.ffi = cffi.FFI() + _egl.ffi.cdef(""" + typedef int EGLint; + typedef unsigned int EGLBoolean; + typedef unsigned int EGLenum; + typedef unsigned int GLenum; + typedef void *EGLContext; + typedef void *EGLDisplay; + typedef void *EGLClientBuffer; + typedef void *EGLImageKHR; + typedef void *GLeglImageOES; + + EGLDisplay eglGetCurrentDisplay(void); + EGLint eglGetError(void); + EGLImageKHR eglCreateImageKHR(EGLDisplay dpy, EGLContext ctx, + EGLenum target, EGLClientBuffer buffer, + const EGLint *attrib_list); + EGLBoolean eglDestroyImageKHR(EGLDisplay dpy, EGLImageKHR image); + void glEGLImageTargetTexture2DOES(GLenum target, GLeglImageOES image); + void glBindTexture(GLenum target, unsigned int texture); + void glActiveTexture(GLenum texture); + """) + + # Load libraries + _egl.egl_lib = _egl.ffi.dlopen("libEGL.so") + _egl.gles_lib = _egl.ffi.dlopen("libGLESv2.so") + + # Cast NULL pointers + _egl.NO_CONTEXT = _egl.ffi.cast("void *", 0) + _egl.NO_DISPLAY = _egl.ffi.cast("void *", 0) + _egl.NO_IMAGE_KHR = _egl.ffi.cast("void *", 0) + + # Bind functions + _egl.get_current_display = _egl.egl_lib.eglGetCurrentDisplay + _egl.create_image_khr = _egl.egl_lib.eglCreateImageKHR + _egl.destroy_image_khr = _egl.egl_lib.eglDestroyImageKHR + _egl.image_target_texture = _egl.gles_lib.glEGLImageTargetTexture2DOES + _egl.get_error = _egl.egl_lib.eglGetError + _egl.bind_texture = _egl.gles_lib.glBindTexture + _egl.active_texture = _egl.gles_lib.glActiveTexture + + # Initialize EGL display once here + _egl.display = _egl.get_current_display() + if _egl.display == _egl.NO_DISPLAY: + raise RuntimeError("Failed to get EGL display") + + _egl.initialized = True + return True + except Exception as e: + cloudlog.exception(f"EGL initialization failed: {e}") + _egl.initialized = False + return False + + +def create_egl_image(width: int, height: int, stride: int, fd: int, uv_offset: int) -> EGLImage | None: + assert _egl.initialized, "EGL not initialized" + + # Duplicate fd since EGL needs it + dup_fd = os.dup(fd) + + # Create image attributes for EGL + img_attrs = [ + EGL_WIDTH, width, + EGL_HEIGHT, height, + EGL_LINUX_DRM_FOURCC_EXT, DRM_FORMAT_NV12, + EGL_DMA_BUF_PLANE0_FD_EXT, dup_fd, + EGL_DMA_BUF_PLANE0_OFFSET_EXT, 0, + EGL_DMA_BUF_PLANE0_PITCH_EXT, stride, + EGL_DMA_BUF_PLANE1_FD_EXT, dup_fd, + EGL_DMA_BUF_PLANE1_OFFSET_EXT, uv_offset, + EGL_DMA_BUF_PLANE1_PITCH_EXT, stride, + EGL_NONE + ] + + attr_array = _egl.ffi.new("int[]", img_attrs) + egl_image = _egl.create_image_khr(_egl.display, _egl.NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, _egl.ffi.NULL, attr_array) + + if egl_image == _egl.NO_IMAGE_KHR: + cloudlog.error(f"Failed to create EGL image: {_egl.get_error()}") + os.close(dup_fd) + return None + + return EGLImage(egl_image=egl_image, fd=dup_fd) + + +def destroy_egl_image(egl_image: EGLImage) -> None: + assert _egl.initialized, "EGL not initialized" + + _egl.destroy_image_khr(_egl.display, egl_image.egl_image) + + # Close the duplicated fd we created in create_egl_image() + # We need to handle OSError since the fd might already be closed + try: + os.close(egl_image.fd) + except OSError: + pass + + +def bind_egl_image_to_texture(texture_id: int, egl_image: EGLImage) -> None: + assert _egl.initialized, "EGL not initialized" + + _egl.active_texture(GL_TEXTURE0) + _egl.bind_texture(GL_TEXTURE_EXTERNAL_OES, texture_id) + _egl.image_target_texture(GL_TEXTURE_EXTERNAL_OES, egl_image.egl_image) diff --git a/system/ui/widgets/cameraview.py b/system/ui/widgets/cameraview.py index 09fe306176..01aac0370b 100644 --- a/system/ui/widgets/cameraview.py +++ b/system/ui/widgets/cameraview.py @@ -1,6 +1,8 @@ import pyray as rl -from msgq.visionipc import VisionIpcClient, VisionStreamType +from openpilot.system.hardware import TICI +from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf from openpilot.system.ui.lib.application import gui_app +from openpilot.system.ui.lib.egl import init_egl, create_egl_image, destroy_egl_image, bind_egl_image_to_texture, EGLImage VERTEX_SHADER = """ @@ -19,30 +21,67 @@ void main() { } """ -FRAME_FRAGMENT_SHADER = """ -#version 300 es -precision mediump float; -in vec2 fragTexCoord; -uniform sampler2D texture0; -uniform sampler2D texture1; -out vec4 fragColor; -void main() { - float y = texture(texture0, fragTexCoord).r; - vec2 uv = texture(texture1, fragTexCoord).ra - 0.5; - fragColor = vec4(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x, 1.0); -} -""" +# Choose fragment shader based on platform capabilities +if TICI: + FRAME_FRAGMENT_SHADER = """ + #version 300 es + #extension GL_OES_EGL_image_external_essl3 : enable + precision mediump float; + in vec2 fragTexCoord; + uniform samplerExternalOES texture0; + out vec4 fragColor; + void main() { + vec4 color = texture(texture0, fragTexCoord); + fragColor = vec4(pow(color.rgb, vec3(1.0/1.28)), color.a); + } + """ +else: + FRAME_FRAGMENT_SHADER = """ + #version 300 es + precision mediump float; + in vec2 fragTexCoord; + uniform sampler2D texture0; + uniform sampler2D texture1; + out vec4 fragColor; + void main() { + float y = texture(texture0, fragTexCoord).r; + vec2 uv = texture(texture1, fragTexCoord).ra - 0.5; + fragColor = vec4(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x, 1.0); + } + """ class CameraView: def __init__(self, name: str, stream_type: VisionStreamType): self.client = VisionIpcClient(name, stream_type, False) self.shader = rl.load_shader_from_memory(VERTEX_SHADER, FRAME_FRAGMENT_SHADER) + + self.frame: VisionBuf | None = None self.texture_y: rl.Texture | None = None self.texture_uv: rl.Texture | None = None - self.frame = None - def close(self): + # EGL resources + self.egl_images: dict[int, EGLImage] = {} + self.egl_texture: rl.Texture | None = None + + # Initialize EGL for zero-copy rendering on TICI + if TICI: + if not init_egl(): + raise RuntimeError("Failed to initialize EGL") + + # Create a 1x1 pixel placeholder texture for EGL image binding + temp_image = rl.gen_image_color(1, 1, rl.BLACK) + self.egl_texture = rl.load_texture_from_image(temp_image) + rl.unload_image(temp_image) + + def close(self) -> None: self._clear_textures() + + # Clean up EGL texture + if TICI and self.egl_texture: + rl.unload_texture(self.egl_texture) + self.egl_texture = None + + # Clean up shader if self.shader and self.shader.id: rl.unload_shader(self.shader) @@ -50,16 +89,13 @@ class CameraView: if not self._ensure_connection(): return + # Try to get a new buffer without blocking buffer = self.client.recv(timeout_ms=0) - self.frame = buffer if buffer else self.frame - if not self.frame or not self.texture_y or not self.texture_uv: - return + if buffer: + self.frame = buffer - y_data = self.frame.data[: self.frame.uv_offset] - uv_data = self.frame.data[self.frame.uv_offset :] - - rl.update_texture(self.texture_y, rl.ffi.cast("void *", y_data.ctypes.data)) - rl.update_texture(self.texture_uv, rl.ffi.cast("void *", uv_data.ctypes.data)) + if not self.frame: + return # Calculate scaling to maintain aspect ratio scale = min(rect.width / self.frame.width, rect.height / self.frame.height) @@ -68,6 +104,53 @@ class CameraView: src_rect = rl.Rectangle(0, 0, float(self.frame.width), float(self.frame.height)) dst_rect = rl.Rectangle(x_offset, y_offset, self.frame.width * scale, self.frame.height * scale) + # Render with appropriate method + if TICI: + self._render_egl(src_rect, dst_rect) + else: + self._render_textures(src_rect, dst_rect) + + def _render_egl(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None: + """Render using EGL for direct buffer access""" + if self.frame is None or self.egl_texture is None: + return + + idx = self.frame.idx + egl_image = self.egl_images.get(idx) + + # Create EGL image if needed + if egl_image is None: + egl_image = create_egl_image(self.frame.width, self.frame.height, self.frame.stride, self.frame.fd, self.frame.uv_offset) + if egl_image: + self.egl_images[idx] = egl_image + else: + return + + # Update texture dimensions to match current frame + self.egl_texture.width = self.frame.width + self.egl_texture.height = self.frame.height + + # Bind the EGL image to our texture + bind_egl_image_to_texture(self.egl_texture.id, egl_image) + + # Render with shader + rl.begin_shader_mode(self.shader) + rl.draw_texture_pro(self.egl_texture, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE) + rl.end_shader_mode() + + def _render_textures(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None: + """Render using texture copies""" + if not self.texture_y or not self.texture_uv or self.frame is None: + return + + # Update textures with new frame data + y_data = self.frame.data[: self.frame.uv_offset] + uv_data = self.frame.data[self.frame.uv_offset :] + + rl.update_texture(self.texture_y, rl.ffi.cast("void *", y_data.ctypes.data)) + rl.update_texture(self.texture_uv, rl.ffi.cast("void *", uv_data.ctypes.data)) + + # Render with shader rl.begin_shader_mode(self.shader) rl.set_shader_value_texture(self.shader, rl.get_shader_location(self.shader, "texture1"), self.texture_uv) rl.draw_texture_pro(self.texture_y, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE) @@ -80,17 +163,30 @@ class CameraView: return False self._clear_textures() - self.texture_y = rl.load_texture_from_image(rl.Image(None, int(self.client.stride), - int(self.client.height), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAYSCALE)) - self.texture_uv = rl.load_texture_from_image(rl.Image(None, int(self.client.stride // 2), - int(self.client.height // 2), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA)) + + if not TICI: + self.texture_y = rl.load_texture_from_image(rl.Image(None, int(self.client.stride), + int(self.client.height), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAYSCALE)) + self.texture_uv = rl.load_texture_from_image(rl.Image(None, int(self.client.stride // 2), + int(self.client.height // 2), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA)) + return True def _clear_textures(self): if self.texture_y and self.texture_y.id: rl.unload_texture(self.texture_y) + self.texture_y = None + if self.texture_uv and self.texture_uv.id: rl.unload_texture(self.texture_uv) + self.texture_uv = None + + # Clean up EGL resources + if TICI: + for data in self.egl_images.values(): + destroy_egl_image(data) + self.egl_images = {} + if __name__ == "__main__": gui_app.init_window("watch3")