using System.Runtime.InteropServices; using Avalonia; using Avalonia.Controls; using Avalonia.LogicalTree; using Avalonia.Platform; using Avalonia.Rendering.Composition; using Avalonia.Threading; using GSS2.Test.ViewModels; using GSS2.Vulkan; using LibCameraSharp; using Silk.NET.Vulkan; using Silk.NET.Vulkan.Extensions.KHR; using Image = Silk.NET.Vulkan.Image; namespace GSS2.Test.Views; public class VulkanImageResources { public Image ImportImage { get; init; } public DeviceMemory ImportMemory { get; init; } public Image ExportImage { get; init; } public DeviceMemory ExportMemory { get; init; } } class VulkanExportedImage { public required int ImageExportedSemaphoreFd { get; init; } public required int ImageRenderedSemaphoreFd { get; init; } public required int? FenceFd { get; init; } public required int OrignalFd { get; init; } public required int ImageFd { get; init; } public required int ImageWidth { get; init; } public required int ImageHeight { get; init; } public required ulong ImageMemorySize { get; init; } public required int WaitSemaphoreFd { get; init; } public required int SignalSemaphoreFd { get; init; } public required VulkanImageResources Resources { get; init; } public (IPlatformHandle imageHandle, PlatformGraphicsExternalImageProperties imageProperties, IPlatformHandle waitSemaphoreHandle, IPlatformHandle signalSemaphoreHandle) Export() { return ( new PlatformHandle(new IntPtr(ImageFd), KnownPlatformGraphicsExternalImageHandleTypes.VulkanOpaquePosixFileDescriptor), new PlatformGraphicsExternalImageProperties { Width = ImageWidth, Height = ImageHeight, Format = PlatformGraphicsExternalImageFormat.B8G8R8A8UNorm, MemorySize = ImageMemorySize }, new PlatformHandle(new IntPtr(WaitSemaphoreFd), KnownPlatformGraphicsExternalSemaphoreHandleTypes.VulkanOpaquePosixFileDescriptor), new PlatformHandle(new IntPtr(SignalSemaphoreFd), KnownPlatformGraphicsExternalSemaphoreHandleTypes.VulkanOpaquePosixFileDescriptor) ); } } class VulkanRenderer { [DllImport("libc")] private static extern int dup(int fd); [DllImport("libc")] private static extern int close(int fd); private readonly VulkanContext _context = new VulkanContext("GSS2.Test", "CameraView"); unsafe public VulkanExportedImage Import(int imageFd, int width, int height, int stride, int? fenceFd, bool initializationFrame) { // Console.WriteLine("Import 1"); var imageFdDuplicate = dup(imageFd); // Console.WriteLine("Import 2"); try { // Console.WriteLine("Import 3"); var imageExportedSemaphore = _context.ImageExportedSemaphore; var imageRenderedSemaphore = _context.ImageRenderedSemaphore; var commandBuffer = _context.CommandBuffer; // Console.WriteLine("Import 7"); var imageExportedSemaphoreGetFdInfoKHR = new SemaphoreGetFdInfoKHR() { SType = StructureType.SemaphoreGetFDInfoKhr, Semaphore = imageExportedSemaphore, HandleType = ExternalSemaphoreHandleTypeFlags.OpaqueFDBit }; _context.KhrExternalSemaphoreFd.GetSemaphoreF(_context.Device, in imageExportedSemaphoreGetFdInfoKHR, out var imageExportedSemaphoreFd).ThrowOnError(); // Console.WriteLine("Import 8"); var imageRenderedSemaphoreGetFdInfoKHR = new SemaphoreGetFdInfoKHR() { SType = StructureType.SemaphoreGetFDInfoKhr, Semaphore = imageRenderedSemaphore, HandleType = ExternalSemaphoreHandleTypeFlags.OpaqueFDBit }; _context.KhrExternalSemaphoreFd.GetSemaphoreF(_context.Device, in imageRenderedSemaphoreGetFdInfoKHR, out var imageRenderedSemaphoreFd).ThrowOnError(); Silk.NET.Vulkan.Fence imageCapturedFence = new Silk.NET.Vulkan.Fence(null); if (fenceFd is not null) { var importSemaphoreFdInfoKHR = new ImportFenceFdInfoKHR() { SType = StructureType.ImportSemaphoreFDInfoKhr, Fd = fenceFd.Value, Flags = FenceImportFlags.TemporaryBit, HandleType = ExternalFenceHandleTypeFlags.OpaqueFDBit }; _context.KhrExternalFenceFd.ImportFenceF(_context.Device, &importSemaphoreFdInfoKHR); } // Console.WriteLine("Import 9"); var (importImage, importMemoryReqirements, importMemory) = ImportImage(imageFdDuplicate, width, height, stride); // Console.WriteLine("Import 10"); var (exportImage, exportMemoryReqirements, exportMemory) = CreateExportImage(width, height); // Console.WriteLine("Import 11"); var commandBufferBeginInfo = new CommandBufferBeginInfo { SType = StructureType.CommandBufferBeginInfo, Flags = CommandBufferUsageFlags.OneTimeSubmitBit }; _context.Api.BeginCommandBuffer(_context.CommandBuffer, &commandBufferBeginInfo).ThrowOnError(); // Console.WriteLine("Import 12"); TransitionImageLayout(_context.CommandBuffer, importImage, ImageLayout.Undefined, ImageLayout.TransferSrcOptimal); TransitionImageLayout(_context.CommandBuffer, exportImage, ImageLayout.Undefined, ImageLayout.TransferDstOptimal); // Console.WriteLine("Import 13"); var imageCopy = new ImageCopy { SrcSubresource = new ImageSubresourceLayers { AspectMask = ImageAspectFlags.ColorBit, MipLevel = 0, BaseArrayLayer = 0, LayerCount = 1 }, SrcOffset = new Offset3D(0, 0, 0), DstSubresource = new ImageSubresourceLayers { AspectMask = ImageAspectFlags.ColorBit, MipLevel = 0, BaseArrayLayer = 0, LayerCount = 1 }, DstOffset = new Offset3D(0, 0, 0), Extent = new Extent3D((uint)width, (uint)height, 1) }; _context.Api.CmdCopyImage(_context.CommandBuffer, importImage, ImageLayout.Undefined, exportImage, ImageLayout.Undefined, [imageCopy]); // Console.WriteLine("Import 14"); TransitionImageLayout(_context.CommandBuffer, exportImage, ImageLayout.TransferDstOptimal, ImageLayout.General); // Console.WriteLine("Import 154"); _context.Api.EndCommandBuffer(_context.CommandBuffer).ThrowOnError(); // Console.WriteLine("Import 16"); var memoryGetFdInfoKHR = new MemoryGetFdInfoKHR { SType = StructureType.MemoryGetFDInfoKhr, Memory = exportMemory, HandleType = ExternalMemoryHandleTypeFlags.OpaqueFDBit }; _context.KhrExternalMemoryFd.GetMemoryF(_context.Device, &memoryGetFdInfoKHR, out var vulkanImageFd).ThrowOnError(); // Console.WriteLine("Import 17"); var waitSemaphoreIndex = 0; Silk.NET.Vulkan.Semaphore[] waitSemaphoresArray = new Silk.NET.Vulkan.Semaphore[ !initializationFrame ? 1 : 0 ]; PipelineStageFlags[] pipelineStageFlagsArray = new PipelineStageFlags[waitSemaphoresArray.Length]; if (!initializationFrame) { waitSemaphoresArray[waitSemaphoreIndex] = imageRenderedSemaphore; pipelineStageFlagsArray[waitSemaphoreIndex++] = PipelineStageFlags.AllGraphicsBit; } var waitSemaphoresSpan = new ReadOnlySpan(waitSemaphoresArray); var pipelineStageFlagsSpan = new ReadOnlySpan(pipelineStageFlagsArray); _context.Api.Semaphore fixed (PipelineStageFlags* pPipelineStageFlagsSpan = pipelineStageFlagsSpan) { fixed (Silk.NET.Vulkan.Semaphore* pWaitSemaphoresSpan = waitSemaphoresSpan) { var submitInfo = new SubmitInfo { SType = StructureType.SubmitInfo, CommandBufferCount = 1, PCommandBuffers = &commandBuffer, PSignalSemaphores = &imageExportedSemaphore, SignalSemaphoreCount = 1, PWaitSemaphores = waitSemaphoresArray.Length == 0 ? null : pWaitSemaphoresSpan, PWaitDstStageMask = waitSemaphoresArray.Length == 0 ? null : pPipelineStageFlagsSpan, WaitSemaphoreCount = checked((uint)waitSemaphoresArray.Length) }; // Console.WriteLine("Import 18"); Console.WriteLine(imageCapturedFence.Handle); _context.Api.QueueSubmit(_context.Queue, 1, &submitInfo, imageCapturedFence).ThrowOnError(); } } // ReadOnlySpan pipelineStageFlags = [PipelineStageFlags.AllGraphicsBit]; // fixed (PipelineStageFlags* pPipelineStageFlags = pipelineStageFlags) // { // ReadOnlySpan waitSemaphores = // [ // imageRenderedSemaphore // ]; // var submitInfo = new SubmitInfo // { // SType = StructureType.SubmitInfo, // CommandBufferCount = 1, // PCommandBuffers = &commandBuffer, // PSignalSemaphores = &imageExportedSemaphore, // SignalSemaphoreCount = 1, // PWaitSemaphores = initializationFrame ? null : &imageRenderedSemaphore, // WaitSemaphoreCount = initializationFrame ? 0u : 1u, // PWaitDstStageMask = initializationFrame ? null : pPipelineStageFlags // }; // // Console.WriteLine("Import 18"); // _context.Api.QueueSubmit(_context.Queue, 1, &submitInfo, new Silk.NET.Vulkan.Fence(UIntPtr.Zero)).ThrowOnError(); // } // Console.WriteLine("Import 19"); return new VulkanExportedImage { ImageExportedSemaphoreFd = imageExportedSemaphoreFd, ImageRenderedSemaphoreFd = imageRenderedSemaphoreFd, FenceFd = fenceFd, OrignalFd = imageFdDuplicate, ImageWidth = width, ImageHeight = height, ImageFd = vulkanImageFd, ImageMemorySize = exportMemoryReqirements.Size, WaitSemaphoreFd = imageExportedSemaphoreFd, SignalSemaphoreFd = imageRenderedSemaphoreFd, Resources = new VulkanImageResources { ImportImage = importImage, ImportMemory = importMemory, ExportImage = exportImage, ExportMemory = exportMemory } }; } catch (Exception e) { // Console.WriteLine(e.Message); // Console.WriteLine(e.StackTrace); throw; } } unsafe public void ClearResource(VulkanExportedImage image) { _context.Api.DestroyImage(_context.Device, image.Resources.ImportImage, null); _context.Api.FreeMemory(_context.Device, image.Resources.ImportMemory, null); _context.Api.DestroyImage(_context.Device, image.Resources.ExportImage, null); _context.Api.FreeMemory(_context.Device, image.Resources.ExportMemory, null); if (image.FenceFd is not null) close(image.FenceFd.Value); close(image.OrignalFd); close(image.ImageFd); close(image.ImageExportedSemaphoreFd); close(image.ImageRenderedSemaphoreFd); } unsafe public (Image, MemoryRequirements, DeviceMemory) ImportImage(int fd, int width, int height, int stride) { var subresourceLayout = new SubresourceLayout { Offset = 0, RowPitch = checked((ulong)stride), }; var imageDrmFormatModifierExplicitCreateInfoEXT = new ImageDrmFormatModifierExplicitCreateInfoEXT { SType = StructureType.ImageDrmFormatModifierExplicitCreateInfoExt, DrmFormatModifier = 0, DrmFormatModifierPlaneCount = 1, PPlaneLayouts = &subresourceLayout, }; var imageCreateInfo = new ImageCreateInfo { SType = StructureType.ImageCreateInfo, PNext = &imageDrmFormatModifierExplicitCreateInfoEXT, ImageType = ImageType.Type2D, Format = Format.B8G8R8A8Unorm, Extent = new Extent3D { Width = checked((uint)width), Height = checked((uint)height), Depth = 1 }, MipLevels = 1, ArrayLayers = 1, Samples = SampleCountFlags.Count1Bit, Tiling = ImageTiling.DrmFormatModifierExt, Usage = ImageUsageFlags.TransferSrcBit | ImageUsageFlags.SampledBit, SharingMode = SharingMode.Exclusive, InitialLayout = ImageLayout.Undefined }; _context.Api.CreateImage(_context.Device, &imageCreateInfo, null, out var image).ThrowOnError(); _context.Api.GetImageMemoryRequirements(_context.Device, image, out var memoryReqirements); var importMemoryFdInfoKHR = new ImportMemoryFdInfoKHR { SType = StructureType.ImportMemoryFDInfoKhr, HandleType = ExternalMemoryHandleTypeFlags.DmaBufBitExt, Fd = fd, }; var memoryAllocateInfo = new MemoryAllocateInfo { SType = StructureType.MemoryAllocateInfo, AllocationSize = memoryReqirements.Size, MemoryTypeIndex = checked((uint)FindMemoryType(memoryReqirements.MemoryTypeBits, MemoryPropertyFlags.DeviceLocalBit)), PNext = &importMemoryFdInfoKHR, }; _context.Api.AllocateMemory(_context.Device, &memoryAllocateInfo, null, out var memory).ThrowOnError(); _context.Api.BindImageMemory(_context.Device, image, memory, 0).ThrowOnError(); return (image, memoryReqirements, memory); } unsafe public (Image, MemoryRequirements, DeviceMemory) CreateExportImage(int width, int height) { var externalMemoryImageCreateInfo = new ExternalMemoryImageCreateInfo { SType = StructureType.ExternalMemoryImageCreateInfo, HandleTypes = ExternalMemoryHandleTypeFlags.OpaqueFDBit }; var imageCreateInfo = new ImageCreateInfo { SType = StructureType.ImageCreateInfo, PNext = &externalMemoryImageCreateInfo, ImageType = ImageType.Type2D, Format = Format.B8G8R8A8Unorm, Extent = new Extent3D { Width = checked((uint)width), Height = checked((uint)height), Depth = 1 }, MipLevels = 1, ArrayLayers = 1, Samples = SampleCountFlags.Count1Bit, Tiling = ImageTiling.Optimal, Usage = ImageUsageFlags.TransferDstBit | ImageUsageFlags.SampledBit, SharingMode = SharingMode.Exclusive, InitialLayout = ImageLayout.Undefined, }; _context.Api.CreateImage(_context.Device, &imageCreateInfo, null, out var image).ThrowOnError(); _context.Api.GetImageMemoryRequirements(_context.Device, image, out var memoryReqirements); var exportMemoryFdInfoKHR = new ExportMemoryAllocateInfoKHR { SType = StructureType.ExportMemoryAllocateInfoKhr, HandleTypes = ExternalMemoryHandleTypeFlags.DmaBufBitExt }; var exportAllocateInfo = new MemoryAllocateInfo { SType = StructureType.MemoryAllocateInfo, AllocationSize = memoryReqirements.Size, MemoryTypeIndex = checked((uint)FindMemoryType(memoryReqirements.MemoryTypeBits, MemoryPropertyFlags.DeviceLocalBit)), PNext = &exportMemoryFdInfoKHR, }; _context.Api.AllocateMemory(_context.Device, &exportAllocateInfo, null, out var memory).ThrowOnError(); _context.Api.BindImageMemory(_context.Device, image, memory, 0).ThrowOnError(); return (image, memoryReqirements, memory); } private int FindMemoryType(uint typeFilter, MemoryPropertyFlags properties) { _context!.Api.GetPhysicalDeviceMemoryProperties(_context.PhysicalDevice, out var memProps); for (int i = 0; i < memProps.MemoryTypeCount; i++) if ((typeFilter & (1 << i)) != 0 && (memProps.MemoryTypes[i].PropertyFlags & properties) == properties) return i; return -1; } unsafe private void TransitionImageLayout(CommandBuffer cmd, Image image, ImageLayout oldLayout, ImageLayout newLayout) { var barrier = new ImageMemoryBarrier { SType = StructureType.ImageMemoryBarrier, OldLayout = oldLayout, NewLayout = newLayout, SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, DstQueueFamilyIndex = Vk.QueueFamilyIgnored, Image = image, SubresourceRange = new ImageSubresourceRange { AspectMask = ImageAspectFlags.ColorBit, BaseMipLevel = 0, LevelCount = 1, BaseArrayLayer = 0, LayerCount = 1 } }; _context.Api.CmdPipelineBarrier( cmd, PipelineStageFlags.TopOfPipeBit, PipelineStageFlags.TransferBit, 0, 0, null, 0, null, 1, &barrier); } } public partial class CameraView : Control { class PresentedFrame { public required ICompositionImportedGpuImage Image { get; init; } public required ICompositionImportedGpuSemaphore Wait { get; init; } public required ICompositionImportedGpuSemaphore Signal { get; init; } public required VulkanExportedImage Vulkan { get; init; } public async ValueTask DisposeAsync(VulkanRenderer? _renerer) { Dispatcher.UIThread.Invoke(() => { try { Image.DisposeAsync().AsTask(); Wait.DisposeAsync().AsTask(); Signal.DisposeAsync().AsTask(); } catch (Exception ex) { Console.WriteLine($"{ex.Message}\n{ex.StackTrace}"); } }); _renerer?.ClearResource(Vulkan); } } private readonly ILogger _logger; private bool _initialized = false; private bool _initializationFrame = true; private Compositor? _compositor; private CompositionSurfaceVisual? _visual; private ICompositionGpuInterop? _gpuInterop; private CompositionDrawingSurface? _surface; private VulkanRenderer? _renerer; private VulkanExportedImage? _latestImage; private PresentedFrame? _lastFrame; private bool _compositionRequested; private System.Timers.Timer? _updateTimer; public CameraView(ILogger logger) { _logger = logger; } protected override void OnAttachedToVisualTree(VisualTreeAttachmentEventArgs e) { base.OnAttachedToVisualTree(e); Initialize(); if (!_initialized) return; int i = 0; _updateTimer = new System.Timers.Timer(105) { AutoReset = false }; _updateTimer.Elapsed += async (_, _) => { await Dispatcher.UIThread.InvokeAsync(async () => { var viewModel = DataContext as CameraViewModel; if (viewModel is null) return; var result = await viewModel.CaptureImage(i++); if (result is not null) { var (frameBuffer, streamConfiguration) = result.Value; // if (frameBuffer.Metadata.Status == FrameMetadata.StatusEnum.FrameSuccess) Update(frameBuffer, streamConfiguration); _updateTimer.Start(); // else // { // var fenceFd = frameBuffer.ReleaseFence()?.Fd?.Release(); // Console.WriteLine($"Buffer returned not success result, closing fd {fenceFd}"); // if (fenceFd is not null) // GSS2.Core.Hardware.CameraService.LibC.close(fenceFd.Value); // } } }); }; _updateTimer.Start(); } protected override void OnDetachedFromLogicalTree(LogicalTreeAttachmentEventArgs e) { if (_initialized) { _updateTimer?.Stop(); _updateTimer?.Dispose(); _surface?.Dispose(); } _initialized = false; base.OnDetachedFromLogicalTree(e); } private async void Initialize() { try { var selfVisual = ElementComposition.GetElementVisual(this)!; _compositor = selfVisual.Compositor; _surface = _compositor.CreateDrawingSurface(); _visual = _compositor.CreateSurfaceVisual(); _visual.Surface = _surface; ElementComposition.SetElementChildVisual(this, _visual); _gpuInterop = await _compositor.TryGetCompositionGpuInterop(); _renerer = new VulkanRenderer(); _initialized = true; _logger.LogInformation("{} initialized", nameof(CameraView)); } catch (Exception ex) { _logger.LogError(ex, "Error while {} initialization", nameof(CameraView)); } } protected override void OnPropertyChanged(AvaloniaPropertyChangedEventArgs change) { if (_visual is not null && change.Property == BoundsProperty) { _visual.Size = new Vector(Bounds.Width, Bounds.Height); Update(); } base.OnPropertyChanged(change); } private void RenderToSurface(VulkanExportedImage image) { var (handle, properties, waitSemaphore, signalSemaphore) = image.Export(); var imageExportedSemaphore = _gpuInterop!.ImportSemaphore(waitSemaphore); imageExportedSemaphore.ImportCompleted.ContinueWith(t => { if (t.IsCompletedSuccessfully) return; _logger.LogError(t.Exception, "Error while imageExported semaphore import"); }); var imageRenderedSemaphore = _gpuInterop!.ImportSemaphore(signalSemaphore); imageExportedSemaphore.ImportCompleted.ContinueWith(t => { if (t.IsCompletedSuccessfully) return; _logger.LogError(t.Exception, "Error while imageRendered semaphore import"); }); var importedImage = _gpuInterop!.ImportImage(handle, properties); imageExportedSemaphore.ImportCompleted.ContinueWith(t => { if (t.IsCompletedSuccessfully) return; _logger.LogError(t.Exception, "Error while image import"); }); var frame = new PresentedFrame { Image = importedImage, Wait = imageExportedSemaphore, Signal = imageRenderedSemaphore, Vulkan = image }; Dispatcher.UIThread.Invoke(async () => { await _surface!.UpdateWithSemaphoresAsync(importedImage, imageExportedSemaphore, imageRenderedSemaphore).ContinueWith(t => { _updateTimer?.Start(); _lastFrame?.DisposeAsync(_renerer); _lastFrame = frame; if (t.IsCompletedSuccessfully) return; _logger.LogError(t.Exception, "Error while image update"); }); }); } private void OnComposition() { _compositionRequested = false; var image = _latestImage; if (image is null) return; RenderToSurface(image); } private void Update(FrameBuffer? frameBuffer = null, StreamConfiguration? streamConfiguration = null) { // Console.WriteLine("Update 1"); if (!_initialized || _compositor is null) return; if (frameBuffer is not null && frameBuffer.Planes.Count() == 1 && streamConfiguration is not null && _renerer is not null) { // Console.WriteLine("Update 2"); var fenceFd = frameBuffer.ReleaseFence()?.Fd?.Release(); Console.WriteLine($"Released fd {fenceFd}"); var image = _renerer.Import( frameBuffer.Planes.First().Fd.Get(), checked((int)streamConfiguration.Size.Width), checked((int)streamConfiguration.Size.Height), checked((int)streamConfiguration.Stride), fenceFd, _initializationFrame); // _renerer.ClearResource(image); // Console.WriteLine("Update 3"); _initializationFrame = false; _latestImage = image; } if (_compositionRequested) return; _compositionRequested = true; _compositor?.RequestCompositionUpdate(OnComposition); } }