sadmen
/
openpilot_comma
mirror of https://github.com/commaai/openpilot.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

system/ui: add EGL support to CameraView (#35338)

Browse Source 
* add EGL support to CameraView

* view 3 cameras

* use a more direct approach

* add new line

* cleanup

* cleanup close()

* extract EGL to a seperate file

* cleanup

* add try/except to close()

* rename egl_textures

* improve implementation
pull/35345/head
Dean Lee 3 months ago committed by GitHub
parent c24f349807
commit 840ced5005
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 301 additions and 28 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 177
      system/ui/lib/egl.py
  2. 152
      system/ui/widgets/cameraview.py

177
system/ui/lib/egl.py
Unescape View File

@ -0,0 +1,177 @@
import os
import cffi
from dataclasses import dataclass
from typing import Any
from openpilot.common.swaglog import cloudlog
# EGL constants
EGL_LINUX_DMA_BUF_EXT = 0x3270
EGL_WIDTH = 0x3057
EGL_HEIGHT = 0x3056
EGL_LINUX_DRM_FOURCC_EXT = 0x3271
EGL_DMA_BUF_PLANE0_FD_EXT = 0x3272
EGL_DMA_BUF_PLANE0_OFFSET_EXT = 0x3273
EGL_DMA_BUF_PLANE0_PITCH_EXT = 0x3274
EGL_DMA_BUF_PLANE1_FD_EXT = 0x3275
EGL_DMA_BUF_PLANE1_OFFSET_EXT = 0x3276
EGL_DMA_BUF_PLANE1_PITCH_EXT = 0x3277
EGL_NONE = 0x3038
GL_TEXTURE0 = 0x84C0
GL_TEXTURE_EXTERNAL_OES = 0x8D65
# DRM Format for NV12
DRM_FORMAT_NV12 = 842094158
@dataclass
class EGLImage:
"""Container for EGL image and associated resources"""
egl_image: Any
fd: int
@dataclass
class EGLState:
"""Container for all EGL-related state"""
initialized: bool = False
ffi: Any = None
egl_lib: Any = None
gles_lib: Any = None
# EGL display connection - shared across all users
display: Any = None
# Constants
NO_CONTEXT: Any = None
NO_DISPLAY: Any = None
NO_IMAGE_KHR: Any = None
# Function pointers
get_current_display: Any = None
create_image_khr: Any = None
destroy_image_khr: Any = None
image_target_texture: Any = None
get_error: Any = None
bind_texture: Any = None
active_texture: Any = None
# Create a single instance of the state
_egl = EGLState()
def init_egl() -> bool:
"""Initialize EGL and load necessary functions"""
global _egl
# Don't re-initialize if already done
if _egl.initialized:
return True
try:
_egl.ffi = cffi.FFI()
_egl.ffi.cdef("""
typedef int EGLint;
typedef unsigned int EGLBoolean;
typedef unsigned int EGLenum;
typedef unsigned int GLenum;
typedef void *EGLContext;
typedef void *EGLDisplay;
typedef void *EGLClientBuffer;
typedef void *EGLImageKHR;
typedef void *GLeglImageOES;
EGLDisplay eglGetCurrentDisplay(void);
EGLint eglGetError(void);
EGLImageKHR eglCreateImageKHR(EGLDisplay dpy, EGLContext ctx,
EGLenum target, EGLClientBuffer buffer,
const EGLint *attrib_list);
EGLBoolean eglDestroyImageKHR(EGLDisplay dpy, EGLImageKHR image);
void glEGLImageTargetTexture2DOES(GLenum target, GLeglImageOES image);
void glBindTexture(GLenum target, unsigned int texture);
void glActiveTexture(GLenum texture);
""")
# Load libraries
_egl.egl_lib = _egl.ffi.dlopen("libEGL.so")
_egl.gles_lib = _egl.ffi.dlopen("libGLESv2.so")
# Cast NULL pointers
_egl.NO_CONTEXT = _egl.ffi.cast("void *", 0)
_egl.NO_DISPLAY = _egl.ffi.cast("void *", 0)
_egl.NO_IMAGE_KHR = _egl.ffi.cast("void *", 0)
# Bind functions
_egl.get_current_display = _egl.egl_lib.eglGetCurrentDisplay
_egl.create_image_khr = _egl.egl_lib.eglCreateImageKHR
_egl.destroy_image_khr = _egl.egl_lib.eglDestroyImageKHR
_egl.image_target_texture = _egl.gles_lib.glEGLImageTargetTexture2DOES
_egl.get_error = _egl.egl_lib.eglGetError
_egl.bind_texture = _egl.gles_lib.glBindTexture
_egl.active_texture = _egl.gles_lib.glActiveTexture
# Initialize EGL display once here
_egl.display = _egl.get_current_display()
if _egl.display == _egl.NO_DISPLAY:
raise RuntimeError("Failed to get EGL display")
_egl.initialized = True
return True
except Exception as e:
cloudlog.exception(f"EGL initialization failed: {e}")
_egl.initialized = False
return False
def create_egl_image(width: int, height: int, stride: int, fd: int, uv_offset: int) -> EGLImage | None:
assert _egl.initialized, "EGL not initialized"
# Duplicate fd since EGL needs it
dup_fd = os.dup(fd)
# Create image attributes for EGL
img_attrs = [
EGL_WIDTH, width,
EGL_HEIGHT, height,
EGL_LINUX_DRM_FOURCC_EXT, DRM_FORMAT_NV12,
EGL_DMA_BUF_PLANE0_FD_EXT, dup_fd,
EGL_DMA_BUF_PLANE0_OFFSET_EXT, 0,
EGL_DMA_BUF_PLANE0_PITCH_EXT, stride,
EGL_DMA_BUF_PLANE1_FD_EXT, dup_fd,
EGL_DMA_BUF_PLANE1_OFFSET_EXT, uv_offset,
EGL_DMA_BUF_PLANE1_PITCH_EXT, stride,
EGL_NONE
]
attr_array = _egl.ffi.new("int[]", img_attrs)
egl_image = _egl.create_image_khr(_egl.display, _egl.NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, _egl.ffi.NULL, attr_array)
if egl_image == _egl.NO_IMAGE_KHR:
cloudlog.error(f"Failed to create EGL image: {_egl.get_error()}")
os.close(dup_fd)
return None
return EGLImage(egl_image=egl_image, fd=dup_fd)
def destroy_egl_image(egl_image: EGLImage) -> None:
assert _egl.initialized, "EGL not initialized"
_egl.destroy_image_khr(_egl.display, egl_image.egl_image)
# Close the duplicated fd we created in create_egl_image()
# We need to handle OSError since the fd might already be closed
try:
os.close(egl_image.fd)
except OSError:
pass
def bind_egl_image_to_texture(texture_id: int, egl_image: EGLImage) -> None:
assert _egl.initialized, "EGL not initialized"
_egl.active_texture(GL_TEXTURE0)
_egl.bind_texture(GL_TEXTURE_EXTERNAL_OES, texture_id)
_egl.image_target_texture(GL_TEXTURE_EXTERNAL_OES, egl_image.egl_image)

152
system/ui/widgets/cameraview.py
Unescape View File

@ -1,6 +1,8 @@
import pyray as rl
from msgq.visionipc import VisionIpcClient, VisionStreamType
from openpilot.system.hardware import TICI
from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf
from openpilot.system.ui.lib.application import gui_app
from openpilot.system.ui.lib.egl import init_egl, create_egl_image, destroy_egl_image, bind_egl_image_to_texture, EGLImage
VERTEX_SHADER = """
@ -19,30 +21,67 @@ void main() {
}
"""
FRAME_FRAGMENT_SHADER = """
#version 300 es
precision mediump float;
in vec2 fragTexCoord;
uniform sampler2D texture0;
uniform sampler2D texture1;
out vec4 fragColor;
void main() {
float y = texture(texture0, fragTexCoord).r;
vec2 uv = texture(texture1, fragTexCoord).ra - 0.5;
fragColor = vec4(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x, 1.0);
}
"""
# Choose fragment shader based on platform capabilities
if TICI:
FRAME_FRAGMENT_SHADER = """
#version 300 es
#extension GL_OES_EGL_image_external_essl3 : enable
precision mediump float;
in vec2 fragTexCoord;
uniform samplerExternalOES texture0;
out vec4 fragColor;
void main() {
vec4 color = texture(texture0, fragTexCoord);
fragColor = vec4(pow(color.rgb, vec3(1.0/1.28)), color.a);
}
"""
else:
FRAME_FRAGMENT_SHADER = """
#version 300 es
precision mediump float;
in vec2 fragTexCoord;
uniform sampler2D texture0;
uniform sampler2D texture1;
out vec4 fragColor;
void main() {
float y = texture(texture0, fragTexCoord).r;
vec2 uv = texture(texture1, fragTexCoord).ra - 0.5;
fragColor = vec4(y + 1.402*uv.y, y - 0.344*uv.x - 0.714*uv.y, y + 1.772*uv.x, 1.0);
}
"""
class CameraView:
def __init__(self, name: str, stream_type: VisionStreamType):
self.client = VisionIpcClient(name, stream_type, False)
self.shader = rl.load_shader_from_memory(VERTEX_SHADER, FRAME_FRAGMENT_SHADER)
self.frame: VisionBuf | None = None
self.texture_y: rl.Texture | None = None
self.texture_uv: rl.Texture | None = None
self.frame = None
def close(self):
# EGL resources
self.egl_images: dict[int, EGLImage] = {}
self.egl_texture: rl.Texture | None = None
# Initialize EGL for zero-copy rendering on TICI
if TICI:
if not init_egl():
raise RuntimeError("Failed to initialize EGL")
# Create a 1x1 pixel placeholder texture for EGL image binding
temp_image = rl.gen_image_color(1, 1, rl.BLACK)
self.egl_texture = rl.load_texture_from_image(temp_image)
rl.unload_image(temp_image)
def close(self) -> None:
self._clear_textures()
# Clean up EGL texture
if TICI and self.egl_texture:
rl.unload_texture(self.egl_texture)
self.egl_texture = None
# Clean up shader
if self.shader and self.shader.id:
rl.unload_shader(self.shader)
@ -50,16 +89,13 @@ class CameraView:
if not self._ensure_connection():
return
# Try to get a new buffer without blocking
buffer = self.client.recv(timeout_ms=0)
self.frame = buffer if buffer else self.frame
if not self.frame or not self.texture_y or not self.texture_uv:
return
if buffer:
self.frame = buffer
y_data = self.frame.data[: self.frame.uv_offset]
uv_data = self.frame.data[self.frame.uv_offset :]
rl.update_texture(self.texture_y, rl.ffi.cast("void *", y_data.ctypes.data))
rl.update_texture(self.texture_uv, rl.ffi.cast("void *", uv_data.ctypes.data))
if not self.frame:
return
# Calculate scaling to maintain aspect ratio
scale = min(rect.width / self.frame.width, rect.height / self.frame.height)
@ -68,6 +104,53 @@ class CameraView:
src_rect = rl.Rectangle(0, 0, float(self.frame.width), float(self.frame.height))
dst_rect = rl.Rectangle(x_offset, y_offset, self.frame.width * scale, self.frame.height * scale)
# Render with appropriate method
if TICI:
self._render_egl(src_rect, dst_rect)
else:
self._render_textures(src_rect, dst_rect)
def _render_egl(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None:
"""Render using EGL for direct buffer access"""
if self.frame is None or self.egl_texture is None:
return
idx = self.frame.idx
egl_image = self.egl_images.get(idx)
# Create EGL image if needed
if egl_image is None:
egl_image = create_egl_image(self.frame.width, self.frame.height, self.frame.stride, self.frame.fd, self.frame.uv_offset)
if egl_image:
self.egl_images[idx] = egl_image
else:
return
# Update texture dimensions to match current frame
self.egl_texture.width = self.frame.width
self.egl_texture.height = self.frame.height
# Bind the EGL image to our texture
bind_egl_image_to_texture(self.egl_texture.id, egl_image)
# Render with shader
rl.begin_shader_mode(self.shader)
rl.draw_texture_pro(self.egl_texture, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
rl.end_shader_mode()
def _render_textures(self, src_rect: rl.Rectangle, dst_rect: rl.Rectangle) -> None:
"""Render using texture copies"""
if not self.texture_y or not self.texture_uv or self.frame is None:
return
# Update textures with new frame data
y_data = self.frame.data[: self.frame.uv_offset]
uv_data = self.frame.data[self.frame.uv_offset :]
rl.update_texture(self.texture_y, rl.ffi.cast("void *", y_data.ctypes.data))
rl.update_texture(self.texture_uv, rl.ffi.cast("void *", uv_data.ctypes.data))
# Render with shader
rl.begin_shader_mode(self.shader)
rl.set_shader_value_texture(self.shader, rl.get_shader_location(self.shader, "texture1"), self.texture_uv)
rl.draw_texture_pro(self.texture_y, src_rect, dst_rect, rl.Vector2(0, 0), 0.0, rl.WHITE)
@ -80,17 +163,30 @@ class CameraView:
return False
self._clear_textures()
self.texture_y = rl.load_texture_from_image(rl.Image(None, int(self.client.stride),
int(self.client.height), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAYSCALE))
self.texture_uv = rl.load_texture_from_image(rl.Image(None, int(self.client.stride // 2),
int(self.client.height // 2), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA))
if not TICI:
self.texture_y = rl.load_texture_from_image(rl.Image(None, int(self.client.stride),
int(self.client.height), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAYSCALE))
self.texture_uv = rl.load_texture_from_image(rl.Image(None, int(self.client.stride // 2),
int(self.client.height // 2), 1, rl.PixelFormat.PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA))
return True
def _clear_textures(self):
if self.texture_y and self.texture_y.id:
rl.unload_texture(self.texture_y)
self.texture_y = None
if self.texture_uv and self.texture_uv.id:
rl.unload_texture(self.texture_uv)
self.texture_uv = None
# Clean up EGL resources
if TICI:
for data in self.egl_images.values():
destroy_egl_image(data)
self.egl_images = {}
if __name__ == "__main__":
gui_app.init_window("watch3")

Write Preview
Loading…
Cancel
Save