feat: CameraView OpenGL render

This commit is contained in:
2026-02-03 20:09:29 +03:00
parent dc3c195c21
commit 99ebb79e0f
9 changed files with 513 additions and 702 deletions
+329 -586
View File
@@ -1,629 +1,372 @@
using System.Runtime.InteropServices;
using Avalonia;
using Avalonia.Controls;
using Avalonia.LogicalTree;
using Avalonia.Platform;
using Avalonia.Rendering.Composition;
using Avalonia.Media;
using Avalonia.OpenGL.Egl;
using Avalonia.OpenGL.Controls;
using Avalonia.OpenGL;
using Avalonia.Threading;
using GSS2.Test.ViewModels;
using GSS2.Vulkan;
using LibCameraSharp;
using Silk.NET.Vulkan;
using Silk.NET.Vulkan.Extensions.KHR;
using Image = Silk.NET.Vulkan.Image;
namespace GSS2.Test.Views;
public class VulkanImageResources
public unsafe partial class CameraView : OpenGlControlBase
{
public Image ImportImage { get; init; }
public DeviceMemory ImportMemory { get; init; }
public Image ExportImage { get; init; }
public DeviceMemory ExportMemory { get; init; }
}
class VulkanExportedImage
{
public required int ImageExportedSemaphoreFd { get; init; }
public required int ImageRenderedSemaphoreFd { get; init; }
public required int? FenceFd { get; init; }
public required int OrignalFd { get; init; }
public required int ImageFd { get; init; }
public required int ImageWidth { get; init; }
public required int ImageHeight { get; init; }
public required ulong ImageMemorySize { get; init; }
public required int WaitSemaphoreFd { get; init; }
public required int SignalSemaphoreFd { get; init; }
public required VulkanImageResources Resources { get; init; }
public (IPlatformHandle imageHandle, PlatformGraphicsExternalImageProperties imageProperties, IPlatformHandle waitSemaphoreHandle, IPlatformHandle signalSemaphoreHandle) Export()
{
return (
new PlatformHandle(new IntPtr(ImageFd), KnownPlatformGraphicsExternalImageHandleTypes.VulkanOpaquePosixFileDescriptor),
new PlatformGraphicsExternalImageProperties
{
Width = ImageWidth,
Height = ImageHeight,
Format = PlatformGraphicsExternalImageFormat.B8G8R8A8UNorm,
MemorySize = ImageMemorySize
},
new PlatformHandle(new IntPtr(WaitSemaphoreFd), KnownPlatformGraphicsExternalSemaphoreHandleTypes.VulkanOpaquePosixFileDescriptor),
new PlatformHandle(new IntPtr(SignalSemaphoreFd), KnownPlatformGraphicsExternalSemaphoreHandleTypes.VulkanOpaquePosixFileDescriptor)
);
}
}
class VulkanRenderer
{
[DllImport("libc")] private static extern int dup(int fd);
[DllImport("libc")] private static extern int close(int fd);
private readonly VulkanContext _context = new VulkanContext("GSS2.Test", "CameraView");
unsafe public VulkanExportedImage Import(int imageFd, int width, int height, int stride, int? fenceFd, bool initializationFrame)
{
// Console.WriteLine("Import 1");
var imageFdDuplicate = dup(imageFd);
// Console.WriteLine("Import 2");
try
{
// Console.WriteLine("Import 3");
var imageExportedSemaphore = _context.ImageExportedSemaphore;
var imageRenderedSemaphore = _context.ImageRenderedSemaphore;
var commandBuffer = _context.CommandBuffer;
// Console.WriteLine("Import 7");
var imageExportedSemaphoreGetFdInfoKHR = new SemaphoreGetFdInfoKHR()
{
SType = StructureType.SemaphoreGetFDInfoKhr,
Semaphore = imageExportedSemaphore,
HandleType = ExternalSemaphoreHandleTypeFlags.OpaqueFDBit
};
_context.KhrExternalSemaphoreFd.GetSemaphoreF(_context.Device, in imageExportedSemaphoreGetFdInfoKHR, out var imageExportedSemaphoreFd).ThrowOnError();
// Console.WriteLine("Import 8");
var imageRenderedSemaphoreGetFdInfoKHR = new SemaphoreGetFdInfoKHR()
{
SType = StructureType.SemaphoreGetFDInfoKhr,
Semaphore = imageRenderedSemaphore,
HandleType = ExternalSemaphoreHandleTypeFlags.OpaqueFDBit
};
_context.KhrExternalSemaphoreFd.GetSemaphoreF(_context.Device, in imageRenderedSemaphoreGetFdInfoKHR, out var imageRenderedSemaphoreFd).ThrowOnError();
Silk.NET.Vulkan.Fence imageCapturedFence = new Silk.NET.Vulkan.Fence(null);
if (fenceFd is not null)
{
var importSemaphoreFdInfoKHR = new ImportFenceFdInfoKHR()
{
SType = StructureType.ImportSemaphoreFDInfoKhr,
Fd = fenceFd.Value,
Flags = FenceImportFlags.TemporaryBit,
HandleType = ExternalFenceHandleTypeFlags.OpaqueFDBit
};
_context.KhrExternalFenceFd.ImportFenceF(_context.Device, &importSemaphoreFdInfoKHR);
}
// Console.WriteLine("Import 9");
var (importImage, importMemoryReqirements, importMemory) = ImportImage(imageFdDuplicate, width, height, stride);
// Console.WriteLine("Import 10");
var (exportImage, exportMemoryReqirements, exportMemory) = CreateExportImage(width, height);
// Console.WriteLine("Import 11");
var commandBufferBeginInfo = new CommandBufferBeginInfo
{
SType = StructureType.CommandBufferBeginInfo,
Flags = CommandBufferUsageFlags.OneTimeSubmitBit
};
_context.Api.BeginCommandBuffer(_context.CommandBuffer, &commandBufferBeginInfo).ThrowOnError();
// Console.WriteLine("Import 12");
TransitionImageLayout(_context.CommandBuffer, importImage, ImageLayout.Undefined, ImageLayout.TransferSrcOptimal);
TransitionImageLayout(_context.CommandBuffer, exportImage, ImageLayout.Undefined, ImageLayout.TransferDstOptimal);
// Console.WriteLine("Import 13");
var imageCopy = new ImageCopy
{
SrcSubresource = new ImageSubresourceLayers
{
AspectMask = ImageAspectFlags.ColorBit,
MipLevel = 0,
BaseArrayLayer = 0,
LayerCount = 1
},
SrcOffset = new Offset3D(0, 0, 0),
DstSubresource = new ImageSubresourceLayers
{
AspectMask = ImageAspectFlags.ColorBit,
MipLevel = 0,
BaseArrayLayer = 0,
LayerCount = 1
},
DstOffset = new Offset3D(0, 0, 0),
Extent = new Extent3D((uint)width, (uint)height, 1)
};
_context.Api.CmdCopyImage(_context.CommandBuffer, importImage, ImageLayout.Undefined, exportImage, ImageLayout.Undefined, [imageCopy]);
// Console.WriteLine("Import 14");
TransitionImageLayout(_context.CommandBuffer, exportImage, ImageLayout.TransferDstOptimal, ImageLayout.General);
// Console.WriteLine("Import 154");
_context.Api.EndCommandBuffer(_context.CommandBuffer).ThrowOnError();
// Console.WriteLine("Import 16");
var memoryGetFdInfoKHR = new MemoryGetFdInfoKHR
{
SType = StructureType.MemoryGetFDInfoKhr,
Memory = exportMemory,
HandleType = ExternalMemoryHandleTypeFlags.OpaqueFDBit
};
_context.KhrExternalMemoryFd.GetMemoryF(_context.Device, &memoryGetFdInfoKHR, out var vulkanImageFd).ThrowOnError();
// Console.WriteLine("Import 17");
var waitSemaphoreIndex = 0;
Silk.NET.Vulkan.Semaphore[] waitSemaphoresArray = new Silk.NET.Vulkan.Semaphore[
!initializationFrame ? 1 : 0
];
PipelineStageFlags[] pipelineStageFlagsArray = new PipelineStageFlags[waitSemaphoresArray.Length];
if (!initializationFrame)
{
waitSemaphoresArray[waitSemaphoreIndex] = imageRenderedSemaphore;
pipelineStageFlagsArray[waitSemaphoreIndex++] = PipelineStageFlags.AllGraphicsBit;
}
var waitSemaphoresSpan = new ReadOnlySpan<Silk.NET.Vulkan.Semaphore>(waitSemaphoresArray);
var pipelineStageFlagsSpan = new ReadOnlySpan<PipelineStageFlags>(pipelineStageFlagsArray);
_context.Api.Semaphore
fixed (PipelineStageFlags* pPipelineStageFlagsSpan = pipelineStageFlagsSpan)
{
fixed (Silk.NET.Vulkan.Semaphore* pWaitSemaphoresSpan = waitSemaphoresSpan)
{
var submitInfo = new SubmitInfo
{
SType = StructureType.SubmitInfo,
CommandBufferCount = 1,
PCommandBuffers = &commandBuffer,
PSignalSemaphores = &imageExportedSemaphore,
SignalSemaphoreCount = 1,
PWaitSemaphores = waitSemaphoresArray.Length == 0 ? null : pWaitSemaphoresSpan,
PWaitDstStageMask = waitSemaphoresArray.Length == 0 ? null : pPipelineStageFlagsSpan,
WaitSemaphoreCount = checked((uint)waitSemaphoresArray.Length)
};
// Console.WriteLine("Import 18");
Console.WriteLine(imageCapturedFence.Handle);
_context.Api.QueueSubmit(_context.Queue, 1, &submitInfo, imageCapturedFence).ThrowOnError();
}
}
// ReadOnlySpan<PipelineStageFlags> pipelineStageFlags = [PipelineStageFlags.AllGraphicsBit];
// fixed (PipelineStageFlags* pPipelineStageFlags = pipelineStageFlags)
// {
// ReadOnlySpan<Silk.NET.Vulkan.Semaphore> waitSemaphores =
// [
// imageRenderedSemaphore
// ];
// var submitInfo = new SubmitInfo
// {
// SType = StructureType.SubmitInfo,
// CommandBufferCount = 1,
// PCommandBuffers = &commandBuffer,
// PSignalSemaphores = &imageExportedSemaphore,
// SignalSemaphoreCount = 1,
// PWaitSemaphores = initializationFrame ? null : &imageRenderedSemaphore,
// WaitSemaphoreCount = initializationFrame ? 0u : 1u,
// PWaitDstStageMask = initializationFrame ? null : pPipelineStageFlags
// };
// // Console.WriteLine("Import 18");
// _context.Api.QueueSubmit(_context.Queue, 1, &submitInfo, new Silk.NET.Vulkan.Fence(UIntPtr.Zero)).ThrowOnError();
// }
// Console.WriteLine("Import 19");
return new VulkanExportedImage
{
ImageExportedSemaphoreFd = imageExportedSemaphoreFd,
ImageRenderedSemaphoreFd = imageRenderedSemaphoreFd,
FenceFd = fenceFd,
OrignalFd = imageFdDuplicate,
ImageWidth = width,
ImageHeight = height,
ImageFd = vulkanImageFd,
ImageMemorySize = exportMemoryReqirements.Size,
WaitSemaphoreFd = imageExportedSemaphoreFd,
SignalSemaphoreFd = imageRenderedSemaphoreFd,
Resources = new VulkanImageResources
{
ImportImage = importImage,
ImportMemory = importMemory,
ExportImage = exportImage,
ExportMemory = exportMemory
}
};
}
catch (Exception e)
{
// Console.WriteLine(e.Message);
// Console.WriteLine(e.StackTrace);
throw;
}
}
unsafe public void ClearResource(VulkanExportedImage image)
{
_context.Api.DestroyImage(_context.Device, image.Resources.ImportImage, null);
_context.Api.FreeMemory(_context.Device, image.Resources.ImportMemory, null);
_context.Api.DestroyImage(_context.Device, image.Resources.ExportImage, null);
_context.Api.FreeMemory(_context.Device, image.Resources.ExportMemory, null);
if (image.FenceFd is not null)
close(image.FenceFd.Value);
close(image.OrignalFd);
close(image.ImageFd);
close(image.ImageExportedSemaphoreFd);
close(image.ImageRenderedSemaphoreFd);
}
unsafe public (Image, MemoryRequirements, DeviceMemory) ImportImage(int fd, int width, int height, int stride)
{
var subresourceLayout = new SubresourceLayout
{
Offset = 0,
RowPitch = checked((ulong)stride),
};
var imageDrmFormatModifierExplicitCreateInfoEXT = new ImageDrmFormatModifierExplicitCreateInfoEXT
{
SType = StructureType.ImageDrmFormatModifierExplicitCreateInfoExt,
DrmFormatModifier = 0,
DrmFormatModifierPlaneCount = 1,
PPlaneLayouts = &subresourceLayout,
};
var imageCreateInfo = new ImageCreateInfo
{
SType = StructureType.ImageCreateInfo,
PNext = &imageDrmFormatModifierExplicitCreateInfoEXT,
ImageType = ImageType.Type2D,
Format = Format.B8G8R8A8Unorm,
Extent = new Extent3D
{
Width = checked((uint)width),
Height = checked((uint)height),
Depth = 1
},
MipLevels = 1,
ArrayLayers = 1,
Samples = SampleCountFlags.Count1Bit,
Tiling = ImageTiling.DrmFormatModifierExt,
Usage = ImageUsageFlags.TransferSrcBit | ImageUsageFlags.SampledBit,
SharingMode = SharingMode.Exclusive,
InitialLayout = ImageLayout.Undefined
};
_context.Api.CreateImage(_context.Device, &imageCreateInfo, null, out var image).ThrowOnError();
_context.Api.GetImageMemoryRequirements(_context.Device, image, out var memoryReqirements);
var importMemoryFdInfoKHR = new ImportMemoryFdInfoKHR
{
SType = StructureType.ImportMemoryFDInfoKhr,
HandleType = ExternalMemoryHandleTypeFlags.DmaBufBitExt,
Fd = fd,
};
var memoryAllocateInfo = new MemoryAllocateInfo
{
SType = StructureType.MemoryAllocateInfo,
AllocationSize = memoryReqirements.Size,
MemoryTypeIndex = checked((uint)FindMemoryType(memoryReqirements.MemoryTypeBits, MemoryPropertyFlags.DeviceLocalBit)),
PNext = &importMemoryFdInfoKHR,
};
_context.Api.AllocateMemory(_context.Device, &memoryAllocateInfo, null, out var memory).ThrowOnError();
_context.Api.BindImageMemory(_context.Device, image, memory, 0).ThrowOnError();
return (image, memoryReqirements, memory);
}
unsafe public (Image, MemoryRequirements, DeviceMemory) CreateExportImage(int width, int height)
{
var externalMemoryImageCreateInfo = new ExternalMemoryImageCreateInfo
{
SType = StructureType.ExternalMemoryImageCreateInfo,
HandleTypes = ExternalMemoryHandleTypeFlags.OpaqueFDBit
};
var imageCreateInfo = new ImageCreateInfo
{
SType = StructureType.ImageCreateInfo,
PNext = &externalMemoryImageCreateInfo,
ImageType = ImageType.Type2D,
Format = Format.B8G8R8A8Unorm,
Extent = new Extent3D
{
Width = checked((uint)width),
Height = checked((uint)height),
Depth = 1
},
MipLevels = 1,
ArrayLayers = 1,
Samples = SampleCountFlags.Count1Bit,
Tiling = ImageTiling.Optimal,
Usage = ImageUsageFlags.TransferDstBit | ImageUsageFlags.SampledBit,
SharingMode = SharingMode.Exclusive,
InitialLayout = ImageLayout.Undefined,
};
_context.Api.CreateImage(_context.Device, &imageCreateInfo, null, out var image).ThrowOnError();
_context.Api.GetImageMemoryRequirements(_context.Device, image, out var memoryReqirements);
var exportMemoryFdInfoKHR = new ExportMemoryAllocateInfoKHR
{
SType = StructureType.ExportMemoryAllocateInfoKhr,
HandleTypes = ExternalMemoryHandleTypeFlags.DmaBufBitExt
};
var exportAllocateInfo = new MemoryAllocateInfo
{
SType = StructureType.MemoryAllocateInfo,
AllocationSize = memoryReqirements.Size,
MemoryTypeIndex = checked((uint)FindMemoryType(memoryReqirements.MemoryTypeBits, MemoryPropertyFlags.DeviceLocalBit)),
PNext = &exportMemoryFdInfoKHR,
};
_context.Api.AllocateMemory(_context.Device, &exportAllocateInfo, null, out var memory).ThrowOnError();
_context.Api.BindImageMemory(_context.Device, image, memory, 0).ThrowOnError();
return (image, memoryReqirements, memory);
}
private int FindMemoryType(uint typeFilter, MemoryPropertyFlags properties)
{
_context!.Api.GetPhysicalDeviceMemoryProperties(_context.PhysicalDevice, out var memProps);
for (int i = 0; i < memProps.MemoryTypeCount; i++)
if ((typeFilter & (1 << i)) != 0 && (memProps.MemoryTypes[i].PropertyFlags & properties) == properties)
return i;
return -1;
}
unsafe private void TransitionImageLayout(CommandBuffer cmd, Image image, ImageLayout oldLayout, ImageLayout newLayout)
{
var barrier = new ImageMemoryBarrier
{
SType = StructureType.ImageMemoryBarrier,
OldLayout = oldLayout,
NewLayout = newLayout,
SrcQueueFamilyIndex = Vk.QueueFamilyIgnored,
DstQueueFamilyIndex = Vk.QueueFamilyIgnored,
Image = image,
SubresourceRange = new ImageSubresourceRange
{
AspectMask = ImageAspectFlags.ColorBit,
BaseMipLevel = 0,
LevelCount = 1,
BaseArrayLayer = 0,
LayerCount = 1
}
};
_context.Api.CmdPipelineBarrier(
cmd,
PipelineStageFlags.TopOfPipeBit,
PipelineStageFlags.TransferBit,
0,
0, null,
0, null,
1, &barrier);
}
}
public partial class CameraView : Control
{
class PresentedFrame
{
public required ICompositionImportedGpuImage Image { get; init; }
public required ICompositionImportedGpuSemaphore Wait { get; init; }
public required ICompositionImportedGpuSemaphore Signal { get; init; }
public required VulkanExportedImage Vulkan { get; init; }
public async ValueTask DisposeAsync(VulkanRenderer? _renerer)
{
Dispatcher.UIThread.Invoke(() =>
{
try
{
Image.DisposeAsync().AsTask();
Wait.DisposeAsync().AsTask();
Signal.DisposeAsync().AsTask();
}
catch (Exception ex)
{
Console.WriteLine($"{ex.Message}\n{ex.StackTrace}");
}
});
_renerer?.ClearResource(Vulkan);
}
}
private readonly ILogger<CameraView> _logger;
private bool _initialized = false;
private bool _initializationFrame = true;
private Compositor? _compositor;
private CompositionSurfaceVisual? _visual;
private ICompositionGpuInterop? _gpuInterop;
private CompositionDrawingSurface? _surface;
private VulkanRenderer? _renerer;
private VulkanExportedImage? _latestImage;
private PresentedFrame? _lastFrame;
private bool _compositionRequested;
private System.Timers.Timer? _updateTimer;
private bool _isEglInitialized = false;
private IntPtr? _eglDisplay = null;
private int? _texture = null;
private int? _program = null;
private int? _uTexLocation = null;
private int? _vao = null;
private int? _vbo = null;
private eglCreateImageKHR_t? _eglCreateImageKHR = null;
private eglDestroyImageKHR_t? _eglDestroyImageKHR = null;
private glEGLImageTargetTexture2DOES_t? _glEGLImageTargetTexture2DOES = null;
private System.Timers.Timer? _renderTimer = null;
public CameraView(ILogger<CameraView> logger)
{
_logger = logger;
_logger.LogInformation("{} initialized", nameof(CameraView));
}
protected override void OnAttachedToVisualTree(VisualTreeAttachmentEventArgs e)
{
(DataContext as CameraViewModel)?.StartViewFinder();
base.OnAttachedToVisualTree(e);
Initialize();
if (!_initialized)
return;
int i = 0;
_updateTimer = new System.Timers.Timer(105)
_renderTimer = new System.Timers.Timer(33)
{
AutoReset = false
AutoReset = true
};
_updateTimer.Elapsed += async (_, _) =>
{
await Dispatcher.UIThread.InvokeAsync(async () =>
{
var viewModel = DataContext as CameraViewModel;
if (viewModel is null)
return;
var result = await viewModel.CaptureImage(i++);
if (result is not null)
{
var (frameBuffer, streamConfiguration) = result.Value;
// if (frameBuffer.Metadata.Status == FrameMetadata.StatusEnum.FrameSuccess)
Update(frameBuffer, streamConfiguration);
_updateTimer.Start();
// else
// {
// var fenceFd = frameBuffer.ReleaseFence()?.Fd?.Release();
// Console.WriteLine($"Buffer returned not success result, closing fd {fenceFd}");
// if (fenceFd is not null)
// GSS2.Core.Hardware.CameraService.LibC.close(fenceFd.Value);
// }
}
});
};
_updateTimer.Start();
_renderTimer.Elapsed += OnRendering;
_renderTimer.Start();
}
protected override void OnDetachedFromLogicalTree(LogicalTreeAttachmentEventArgs e)
{
if (_initialized)
{
_updateTimer?.Stop();
_updateTimer?.Dispose();
_surface?.Dispose();
}
_initialized = false;
(DataContext as CameraViewModel)?.StopViewFinder();
base.OnDetachedFromLogicalTree(e);
}
private async void Initialize()
private void OnRendering(object? sender, EventArgs e)
{
Dispatcher.UIThread.Invoke(() =>
{
RequestNextFrameRendering();
});
}
[DllImport("libEGL.so.1")] private static extern IntPtr eglQueryString(IntPtr dpy, int name);
[DllImport("libEGL.so.1")] private static extern IntPtr eglGetCurrentDisplay();
[DllImport("libEGL.so.1")] static extern IntPtr eglGetProcAddress(string procname);
[DllImport("libGL.so.1")] static extern void glUniform1i(int location, int falue);
delegate IntPtr glEGLImageTargetTexture2DOES_t(int target, IntPtr image);
delegate IntPtr eglCreateImageKHR_t(IntPtr dpy, IntPtr ctx, int target, IntPtr buffer, int[] attribs);
delegate bool eglDestroyImageKHR_t(IntPtr dpy, IntPtr image);
private const uint DRM_FORMAT_BGRX8888 = 0x34325258; // XR24 XRGB8888
private const int EGL_LINUX_DMA_BUF_EXT = 0x3270;
private const int EGL_DMA_BUF_PLANE0_FD_EXT = 0x3272;
private const int EGL_DMA_BUF_PLANE0_OFFSET_EXT = 0x3273;
private const int EGL_DMA_BUF_PLANE0_PITCH_EXT = 0x3274;
private const int EGL_LINUX_DRM_FOURCC_EXT = 0x3271;
private const int GL_CLAMP_TO_EDGE = 0x812F;
private const int GL_TEXTURE_WRAP_S = 0x2802;
private const int GL_TEXTURE_WRAP_T = 0x2803;
private const int GL_TRIANGLE_STRIP = 0x0005;
private const string VertexShader = @"
#version 300 es
layout (location = 0) in vec2 aPos;
layout (location = 1) in vec2 aUv;
out vec2 vUv;
void main()
{
vUv = aUv;
gl_Position = vec4(aPos, 0.0, 1.0);
}
";
private const string FragmentShader = @"
#version 300 es
precision mediump float;
in vec2 vUv;
uniform sampler2D uTex;
out vec4 fragColor;
void main()
{
fragColor = texture(uTex, vUv);
}
";
private readonly float[] Vertices =
[
-1f, -1f, 0f, 1f,
1f, -1f, 1f, 1f,
-1f, 1f, 0f, 0f,
1f, 1f, 1f, 0f,
];
protected override void OnOpenGlInit(GlInterface gl)
{
_logger.LogInformation("OpenGL initialized");
_logger.LogInformation("GL version: {}", gl.GetString(GlConsts.GL_VERSION));
_logger.LogInformation("GL vendor: {}", gl.GetString(GlConsts.GL_VENDOR));
_logger.LogInformation("GL renderer: {}", gl.GetString(GlConsts.GL_RENDERER));
var glExtensions = gl.GetString(GlConsts.GL_EXTENSIONS);
if (glExtensions is null)
{
_logger.LogCritical("Cannot use EGL, GL extentions query returned nullptr");
base.OnOpenGlInit(gl);
return;
}
_logger.LogInformation("GL extentions: [\n\t{}\n]", glExtensions.Trim().Replace(" ", "\n\t"));
if (!glExtensions.Contains("GL_OES_EGL_image"))
{
_logger.LogWarning("Required GL extention \"GL_OES_EGL_image\" not found");
if (!glExtensions.Contains("GL_OES_EGL_image_external"))
{
_logger.LogCritical("Required GL extention \"GL_OES_EGL_image_external\" also not found");
base.OnOpenGlInit(gl);
return;
}
}
_eglDisplay = eglGetCurrentDisplay();
if (_eglDisplay == IntPtr.Zero)
{
_logger.LogCritical("Cannot use EGL, current EGL display is nullptr");
base.OnOpenGlInit(gl);
return;
}
_logger.LogInformation("EGL display: 0x{:X}", _eglDisplay);
var eglExtensions = Marshal.PtrToStringAnsi(eglQueryString(_eglDisplay.Value, EglConsts.EGL_EXTENSIONS));
if (eglExtensions is null)
{
_logger.LogCritical("Cannot use EGL, EGL extentions query returned nullptr");
base.OnOpenGlInit(gl);
return;
}
_logger.LogInformation("EGL extensions: [\n\t{}\n]", eglExtensions.Trim().Replace(" ", "\n\t"));
if (!eglExtensions.Contains("EGL_EXT_image_dma_buf_import"))
{
_logger.LogCritical("Cannot use EGL, required EGL extention \"EGL_EXT_image_dma_buf_import\" not found");
base.OnOpenGlInit(gl);
return;
}
var pEglCreateImageKHR = eglGetProcAddress("eglCreateImageKHR");
if (pEglCreateImageKHR == IntPtr.Zero)
{
_logger.LogCritical("Cannot use EGL, required EGL method \"eglCreateImageKHR\" not found");
base.OnOpenGlInit(gl);
return;
}
_eglCreateImageKHR = Marshal.GetDelegateForFunctionPointer<eglCreateImageKHR_t>(pEglCreateImageKHR);
var pEglDestroyImageKHR = eglGetProcAddress("eglDestroyImageKHR");
if (pEglDestroyImageKHR == IntPtr.Zero)
{
_logger.LogCritical("Cannot use EGL, required EGL method \"eglDestroyImageKHR\" not found");
base.OnOpenGlInit(gl);
return;
}
_eglDestroyImageKHR = Marshal.GetDelegateForFunctionPointer<eglDestroyImageKHR_t>(pEglDestroyImageKHR);
var pGlEGLImageTargetTexture2DOES = eglGetProcAddress("glEGLImageTargetTexture2DOES");
if (pGlEGLImageTargetTexture2DOES == IntPtr.Zero)
{
_logger.LogCritical("Cannot use EGL, required EGL method \"glEGLImageTargetTexture2DOES\" not found");
base.OnOpenGlInit(gl);
return;
}
_glEGLImageTargetTexture2DOES = Marshal.GetDelegateForFunctionPointer<glEGLImageTargetTexture2DOES_t>(pGlEGLImageTargetTexture2DOES);
_texture = gl.GenTexture();
gl.BindTexture(GlConsts.GL_TEXTURE_2D, _texture.Value);
gl.TexParameteri(GlConsts.GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
gl.TexParameteri(GlConsts.GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
gl.TexParameteri(GlConsts.GL_TEXTURE_2D, GlConsts.GL_TEXTURE_MIN_FILTER, GlConsts.GL_LINEAR);
gl.TexParameteri(GlConsts.GL_TEXTURE_2D, GlConsts.GL_TEXTURE_MAG_FILTER, GlConsts.GL_LINEAR);
int vs = gl.CreateShader(GlConsts.GL_VERTEX_SHADER);
gl.ShaderSourceString(vs, VertexShader);
gl.CompileShader(vs);
int fs = gl.CreateShader(GlConsts.GL_FRAGMENT_SHADER);
gl.ShaderSourceString(fs, FragmentShader);
gl.CompileShader(fs);
_program = gl.CreateProgram();
gl.AttachShader(_program.Value, vs);
gl.AttachShader(_program.Value, fs);
gl.LinkProgram(_program.Value);
gl.DeleteShader(vs);
gl.DeleteShader(fs);
_uTexLocation = gl.GetUniformLocationString(_program.Value, "uTex");
_vao = gl.GenVertexArray();
_vbo = gl.GenBuffer();
gl.BindVertexArray(_vao.Value);
gl.BindBuffer(GlConsts.GL_ARRAY_BUFFER, _vbo.Value);
fixed (float* pVertices = Vertices)
gl.BufferData(GlConsts.GL_ARRAY_BUFFER, Vertices.Length * sizeof(float), new IntPtr(pVertices), GlConsts.GL_STATIC_DRAW);
// position
gl.EnableVertexAttribArray(0);
gl.VertexAttribPointer(0, 2, GlConsts.GL_FLOAT, 0, 4 * sizeof(float), IntPtr.Zero);
// uv
gl.EnableVertexAttribArray(1);
gl.VertexAttribPointer(1, 2, GlConsts.GL_FLOAT, 0, 4 * sizeof(float), 2 * sizeof(float));
gl.BindVertexArray(0);
_isEglInitialized = true;
}
protected override void OnOpenGlRender(GlInterface gl, int fb)
{
if (!_isEglInitialized ||
_eglDisplay is null ||
_eglDisplay.Value == IntPtr.Zero ||
_texture is null ||
_program is null ||
_uTexLocation is null ||
_vao is null ||
_vbo is null)
{
_logger.LogError("OpenGL render called with uninitialized EGL");
return;
}
try
{
var selfVisual = ElementComposition.GetElementVisual(this)!;
_compositor = selfVisual.Compositor;
(LibCameraSharp.FrameBuffer, LibCameraSharp.StreamConfiguration)? frame = null;
do
{
frame = (DataContext as CameraViewModel)?.GrabViewFinderFrame();
} while (frame is null);
if (frame is null)
return;
var (frameBuffer, streamConfiguration) = frame.Value;
_surface = _compositor.CreateDrawingSurface();
_visual = _compositor.CreateSurfaceVisual();
_visual.Surface = _surface;
ElementComposition.SetElementChildVisual(this, _visual);
_gpuInterop = await _compositor.TryGetCompositionGpuInterop();
_renerer = new VulkanRenderer();
_initialized = true;
_logger.LogInformation("{} initialized", nameof(CameraView));
// 0. Проверка что полученый фрейм ожидаемый праильный формат
var width = streamConfiguration.Size.Width;
if (width <= 0)
{
_logger.LogError("Cannot render frame, stream configuration vaildation error, expected frame width >= 0, got {}", streamConfiguration.Size.Width);
(DataContext as CameraViewModel)?.ReturnViewFinderFrame(frameBuffer);
return;
}
var height = streamConfiguration.Size.Height;
if (height <= 0)
{
_logger.LogError("Cannot render frame, stream configuration vaildation error, expected frame height >= 0, got {}", streamConfiguration.Size.Height);
(DataContext as CameraViewModel)?.ReturnViewFinderFrame(frameBuffer);
return;
}
if (streamConfiguration.PixelFormat.Fourcc != DRM_FORMAT_BGRX8888)
{
_logger.LogError("Cannot render frame, stream configuration vaildation error, expected forcc: 0x{:X}, got 0x{:X}", DRM_FORMAT_BGRX8888, streamConfiguration.PixelFormat.Fourcc);
(DataContext as CameraViewModel)?.ReturnViewFinderFrame(frameBuffer);
return;
}
if (streamConfiguration.PixelFormat.Modifier != 0)
{
_logger.LogError("Cannot render frame, stream configuration vaildation error, expected DRM modifier: 0x{:X}, got 0x{:X}", 0, streamConfiguration.PixelFormat.Modifier);
(DataContext as CameraViewModel)?.ReturnViewFinderFrame(frameBuffer);
return;
}
if (frameBuffer.Planes.Count() != 1)
{
_logger.LogError("Cannot render frame, planes vaildation error, expected planes count: 0, got {}", frameBuffer.Planes.Count());
(DataContext as CameraViewModel)?.ReturnViewFinderFrame(frameBuffer);
return;
}
var plane = frameBuffer.Planes.First();
if (plane.Length != width * height * 4)
{
_logger.LogError("Cannot render frame, plane validation error, expected plane size 0x{:X}, got 0x{:X}", width * height * 4, plane.Length);
(DataContext as CameraViewModel)?.ReturnViewFinderFrame(frameBuffer);
return;
}
var fd = plane.Fd.Get();
if (fd <= 0)
{
_logger.LogError("Cannot render frame, FD vaildation error, FD >= 0, got 0x{:X}", fd);
(DataContext as CameraViewModel)?.ReturnViewFinderFrame(frameBuffer);
return;
}
// 1. Очистка экрана
gl.Viewport(0, 0, (int)Bounds.Width, (int)Bounds.Height);
gl.ClearColor(
Colors.CornflowerBlue.R / 255f,
Colors.CornflowerBlue.G / 255f,
Colors.CornflowerBlue.B / 255f,
Colors.CornflowerBlue.A / 255f
);
gl.Clear(GlConsts.GL_COLOR_BUFFER_BIT);
// 2. Создание EGLImage из dma-buf
var eglImage = CreateEglImageFromDmabuf(_eglDisplay.Value, fd, (int)width, (int)height, (int)(width * 4));
// 3. Биндинг текстуры
gl.ActiveTexture(GlConsts.GL_TEXTURE0);
gl.BindTexture(GlConsts.GL_TEXTURE_2D, _texture.Value);
// 4. Привязка EGLImage к текстуре
_glEGLImageTargetTexture2DOES!(GlConsts.GL_TEXTURE_2D, eglImage);
// 5. Запуск шейдера
gl.UseProgram(_program.Value);
glUniform1i(_uTexLocation.Value, 0);
// 6. Отрисовка
gl.BindVertexArray(_vao.Value);
gl.DrawArrays(GL_TRIANGLE_STRIP, 0, 4);
// 7. Уничтожение EGLImage
_eglDestroyImageKHR!(_eglDisplay.Value, eglImage);
(DataContext as CameraViewModel)?.ReturnViewFinderFrame(frameBuffer);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error while {} initialization", nameof(CameraView));
_logger.LogError(ex, "Exception whiel OpenGL render");
}
}
protected override void OnPropertyChanged(AvaloniaPropertyChangedEventArgs change)
private IntPtr CreateEglImageFromDmabuf(IntPtr eglDisplay, int fd, int width, int height, int stride)
{
if (_visual is not null && change.Property == BoundsProperty)
var attribs = new[]
{
_visual.Size = new Vector(Bounds.Width, Bounds.Height);
Update();
}
base.OnPropertyChanged(change);
}
private void RenderToSurface(VulkanExportedImage image)
{
var (handle, properties, waitSemaphore, signalSemaphore) = image.Export();
var imageExportedSemaphore = _gpuInterop!.ImportSemaphore(waitSemaphore);
imageExportedSemaphore.ImportCompleted.ContinueWith(t =>
{
if (t.IsCompletedSuccessfully)
return;
_logger.LogError(t.Exception, "Error while imageExported semaphore import");
});
var imageRenderedSemaphore = _gpuInterop!.ImportSemaphore(signalSemaphore);
imageExportedSemaphore.ImportCompleted.ContinueWith(t =>
{
if (t.IsCompletedSuccessfully)
return;
_logger.LogError(t.Exception, "Error while imageRendered semaphore import");
});
var importedImage = _gpuInterop!.ImportImage(handle, properties);
imageExportedSemaphore.ImportCompleted.ContinueWith(t =>
{
if (t.IsCompletedSuccessfully)
return;
_logger.LogError(t.Exception, "Error while image import");
});
var frame = new PresentedFrame
{
Image = importedImage,
Wait = imageExportedSemaphore,
Signal = imageRenderedSemaphore,
Vulkan = image
EglConsts.EGL_WIDTH, width,
EglConsts.EGL_HEIGHT, height,
EGL_LINUX_DRM_FOURCC_EXT, unchecked((int)DRM_FORMAT_BGRX8888),
EGL_DMA_BUF_PLANE0_FD_EXT, fd,
EGL_DMA_BUF_PLANE0_OFFSET_EXT, 0,
EGL_DMA_BUF_PLANE0_PITCH_EXT, stride,
EglConsts.EGL_NONE
};
Dispatcher.UIThread.Invoke(async () =>
{
await _surface!.UpdateWithSemaphoresAsync(importedImage, imageExportedSemaphore, imageRenderedSemaphore).ContinueWith(t =>
{
_updateTimer?.Start();
_lastFrame?.DisposeAsync(_renerer);
_lastFrame = frame;
if (t.IsCompletedSuccessfully)
return;
_logger.LogError(t.Exception, "Error while image update");
});
});
}
private void OnComposition()
{
_compositionRequested = false;
var image = _eglCreateImageKHR?.Invoke(eglDisplay, IntPtr.Zero, EGL_LINUX_DMA_BUF_EXT, IntPtr.Zero, attribs);
var image = _latestImage;
if (image is null)
return;
if (image is null || image == IntPtr.Zero)
throw new Exception("eglCreateImageKHR failed");
RenderToSurface(image);
}
private void Update(FrameBuffer? frameBuffer = null, StreamConfiguration? streamConfiguration = null)
{
// Console.WriteLine("Update 1");
if (!_initialized || _compositor is null)
return;
if (frameBuffer is not null &&
frameBuffer.Planes.Count() == 1 &&
streamConfiguration is not null &&
_renerer is not null)
{
// Console.WriteLine("Update 2");
var fenceFd = frameBuffer.ReleaseFence()?.Fd?.Release();
Console.WriteLine($"Released fd {fenceFd}");
var image = _renerer.Import(
frameBuffer.Planes.First().Fd.Get(),
checked((int)streamConfiguration.Size.Width),
checked((int)streamConfiguration.Size.Height),
checked((int)streamConfiguration.Stride),
fenceFd,
_initializationFrame);
// _renerer.ClearResource(image);
// Console.WriteLine("Update 3");
_initializationFrame = false;
_latestImage = image;
}
if (_compositionRequested)
return;
_compositionRequested = true;
_compositor?.RequestCompositionUpdate(OnComposition);
return image.Value;
}
}