using System.Runtime.InteropServices; using System.Security.Cryptography; using Avalonia; using Avalonia.Controls; using Avalonia.LogicalTree; using Avalonia.Platform; using Avalonia.Rendering.Composition; using Avalonia.Threading; using GSS2.Vulkan; using LibCameraSharp; using Silk.NET.Vulkan; using Silk.NET.Vulkan.Extensions.KHR; using Image = Silk.NET.Vulkan.Image; namespace GSS2.Test; interface IImage { public (IPlatformHandle handle, PlatformGraphicsExternalImageProperties properties, Silk.NET.Vulkan.Semaphore semaphore) Export(); } interface IRenderer { public abstract IImage ImportOrUpdate(int fd, int width, int height); } class VkImage { public int Fd { get; set; } public int Width { get; set; } public int Height { get; set; } public ulong MemorySize { get; set; } public Silk.NET.Vulkan.Semaphore Semaphore { get; set; } public (IPlatformHandle handle, PlatformGraphicsExternalImageProperties properties, Silk.NET.Vulkan.Semaphore semaphore) Export() { return ( new PlatformHandle(new IntPtr(Fd), KnownPlatformGraphicsExternalImageHandleTypes.VulkanOpaquePosixFileDescriptor), new PlatformGraphicsExternalImageProperties { Width = Width, Height = Height, Format = PlatformGraphicsExternalImageFormat.B8G8R8A8UNorm, MemorySize = MemorySize }, Semaphore ); } } class VkRenderer { [DllImport("libc")] private static extern int dup(int fd); readonly VulkanContext context = new VulkanContext("GSS2.Test", "CameraView"); public IPlatformHandle ExportSemaphore(bool renderFinished) { int fd = context.ExportSemaphoreFd(renderFinished ? context.RenderFinishedSemaphore : context.ImageAvailableSemaphore); return new PlatformHandle(new IntPtr(fd), KnownPlatformGraphicsExternalImageHandleTypes.VulkanOpaquePosixFileDescriptor); } unsafe public VkImage ImportOrUpdate(int fd, int width, int height, int stride) { var fdDup = dup(fd); try { FenceCreateInfo fenceInfo = new FenceCreateInfo { SType = StructureType.FenceCreateInfo }; context.Api.CreateFence(context.Device, &fenceInfo, null, out var fence).ThrowOnError(); var (importImage, importMemoryReqirements, importMemory) = ImportImage(fdDup, width, height, stride); var (exportImage, exportMemoryReqirements, exportMemory) = CreateExportImage(width, height); var commandPoolCreateInfo = new CommandPoolCreateInfo { SType = StructureType.CommandPoolCreateInfo, Flags = CommandPoolCreateFlags.ResetCommandBufferBit, QueueFamilyIndex = context.QueueFamilyIndex }; context.Api.CreateCommandPool(context.Device, &commandPoolCreateInfo, null, out var commandPool).ThrowOnError(); var commandBufferAllocateInfo = new CommandBufferAllocateInfo { SType = StructureType.CommandBufferAllocateInfo, CommandPool = commandPool, Level = CommandBufferLevel.Primary, CommandBufferCount = 1 }; context.Api.AllocateCommandBuffers(context.Device, &commandBufferAllocateInfo, out var commandBuffers).ThrowOnError(); var commandBufferBeginInfo = new CommandBufferBeginInfo { SType = StructureType.CommandBufferBeginInfo, Flags = CommandBufferUsageFlags.OneTimeSubmitBit }; context.Api.BeginCommandBuffer(commandBuffers, &commandBufferBeginInfo).ThrowOnError(); TransitionImageLayout(commandBuffers, importImage, ImageLayout.Undefined, ImageLayout.TransferSrcOptimal); TransitionImageLayout(commandBuffers, exportImage, ImageLayout.Undefined, ImageLayout.TransferDstOptimal); var region = new ImageCopy { SrcSubresource = new ImageSubresourceLayers { AspectMask = ImageAspectFlags.ColorBit, MipLevel = 0, BaseArrayLayer = 0, LayerCount = 1 }, SrcOffset = new Offset3D(0, 0, 0), DstSubresource = new ImageSubresourceLayers { AspectMask = ImageAspectFlags.ColorBit, MipLevel = 0, BaseArrayLayer = 0, LayerCount = 1 }, DstOffset = new Offset3D(0, 0, 0), Extent = new Extent3D((uint)width, (uint)height, 1) }; context.Api.CmdCopyImage(commandBuffers, importImage, ImageLayout.Undefined, exportImage, ImageLayout.Undefined, [region]); TransitionImageLayout(commandBuffers, exportImage, ImageLayout.TransferDstOptimal, ImageLayout.General); context.Api.EndCommandBuffer(commandBuffers).ThrowOnError(); if (context.Api.TryGetDeviceExtension(context.Instance, context.Device, out var khrExternalMemoryFd)) Console.WriteLine("KhrExternalMemoryFd import success"); else throw new Exception("KhrExternalMemoryFd import error"); var memoryGetFdInfoKHR = new MemoryGetFdInfoKHR { SType = StructureType.MemoryGetFDInfoKhr, Memory = exportMemory, HandleType = ExternalMemoryHandleTypeFlags.OpaqueFDBit }; khrExternalMemoryFd.GetMemoryF(context.Device, &memoryGetFdInfoKHR, out var vulkanFd).ThrowOnError(); Console.WriteLine($"Image export success {vulkanFd}"); var renderFinished = context.RenderFinishedSemaphore; var submitInfo = new SubmitInfo { SType = StructureType.SubmitInfo, CommandBufferCount = 1, PCommandBuffers = &commandBuffers, PSignalSemaphores = &renderFinished, SignalSemaphoreCount = 1 }; context.Api.QueueSubmit(context.Queue, 1, &submitInfo, fence).ThrowOnError(); context.Api.WaitForFences(context.Device, 1, &fence, true, ulong.MaxValue).ThrowOnError(); context.Api.ResetFences(context.Device, 1, &fence).ThrowOnError(); return new VkImage() { Width = width, Height = height, Fd = vulkanFd, MemorySize = exportMemoryReqirements.Size }; } catch (Exception e) { Console.WriteLine(e.Message); Console.WriteLine(e.StackTrace); throw; } } unsafe public (Image, MemoryRequirements, DeviceMemory) ImportImage(int fd, int width, int height, int stride) { var subresourceLayout = new SubresourceLayout { Offset = 0, RowPitch = checked((ulong)stride), }; var imageDrmFormatModifierExplicitCreateInfoEXT = new ImageDrmFormatModifierExplicitCreateInfoEXT { SType = StructureType.ImageDrmFormatModifierExplicitCreateInfoExt, DrmFormatModifier = 0, DrmFormatModifierPlaneCount = 1, PPlaneLayouts = &subresourceLayout, }; var imageCreateInfo = new ImageCreateInfo { SType = StructureType.ImageCreateInfo, PNext = &imageDrmFormatModifierExplicitCreateInfoEXT, ImageType = ImageType.Type2D, Format = Format.B8G8R8A8Unorm, Extent = new Extent3D { Width = checked((uint)width), Height = checked((uint)height), Depth = 1 }, MipLevels = 1, ArrayLayers = 1, Samples = SampleCountFlags.Count1Bit, Tiling = ImageTiling.DrmFormatModifierExt, Usage = ImageUsageFlags.TransferSrcBit | ImageUsageFlags.SampledBit, SharingMode = SharingMode.Exclusive, InitialLayout = ImageLayout.Undefined }; context.Api.CreateImage(context.Device, &imageCreateInfo, null, out var image).ThrowOnError(); context.Api.GetImageMemoryRequirements(context.Device, image, out var memoryReqirements); var importMemoryFdInfoKHR = new ImportMemoryFdInfoKHR { SType = StructureType.ImportMemoryFDInfoKhr, HandleType = ExternalMemoryHandleTypeFlags.DmaBufBitExt, Fd = fd, }; var memoryAllocateInfo = new MemoryAllocateInfo { SType = StructureType.MemoryAllocateInfo, AllocationSize = memoryReqirements.Size, MemoryTypeIndex = checked((uint)FindMemoryType(memoryReqirements.MemoryTypeBits, MemoryPropertyFlags.DeviceLocalBit)), PNext = &importMemoryFdInfoKHR, }; context.Api.AllocateMemory(context.Device, &memoryAllocateInfo, null, out var memory).ThrowOnError(); context.Api.BindImageMemory(context.Device, image, memory, 0).ThrowOnError(); return (image, memoryReqirements, memory); } unsafe public (Image, MemoryRequirements, DeviceMemory) CreateExportImage(int width, int height) { var externalMemoryImageCreateInfo = new ExternalMemoryImageCreateInfo { SType = StructureType.ExternalMemoryImageCreateInfo, HandleTypes = ExternalMemoryHandleTypeFlags.OpaqueFDBit }; var imageCreateInfo = new ImageCreateInfo { SType = StructureType.ImageCreateInfo, PNext = &externalMemoryImageCreateInfo, ImageType = ImageType.Type2D, Format = Format.B8G8R8A8Unorm, Extent = new Extent3D { Width = checked((uint)width), Height = checked((uint)height), Depth = 1 }, MipLevels = 1, ArrayLayers = 1, Samples = SampleCountFlags.Count1Bit, Tiling = ImageTiling.Optimal, Usage = ImageUsageFlags.TransferDstBit | ImageUsageFlags.SampledBit, SharingMode = SharingMode.Exclusive, InitialLayout = ImageLayout.Undefined, }; context.Api.CreateImage(context.Device, &imageCreateInfo, null, out var image).ThrowOnError(); context.Api.GetImageMemoryRequirements(context.Device, image, out var memoryReqirements); var exportMemoryFdInfoKHR = new ExportMemoryAllocateInfoKHR { SType = StructureType.ExportMemoryAllocateInfoKhr, HandleTypes = ExternalMemoryHandleTypeFlags.DmaBufBitExt }; var exportAllocateInfo = new MemoryAllocateInfo { SType = StructureType.MemoryAllocateInfo, AllocationSize = memoryReqirements.Size, MemoryTypeIndex = checked((uint)FindMemoryType(memoryReqirements.MemoryTypeBits, MemoryPropertyFlags.DeviceLocalBit)), PNext = &exportMemoryFdInfoKHR, }; context.Api.AllocateMemory(context.Device, &exportAllocateInfo, null, out var memory).ThrowOnError(); context.Api.BindImageMemory(context.Device, image, memory, 0).ThrowOnError(); return (image, memoryReqirements, memory); } private int FindMemoryType(uint typeFilter, MemoryPropertyFlags properties) { context!.Api.GetPhysicalDeviceMemoryProperties(context.PhysicalDevice, out var memProps); for (int i = 0; i < memProps.MemoryTypeCount; i++) if ((typeFilter & (1 << i)) != 0 && (memProps.MemoryTypes[i].PropertyFlags & properties) == properties) return i; return -1; } unsafe private void TransitionImageLayout(CommandBuffer cmd, Image image, ImageLayout oldLayout, ImageLayout newLayout) { var barrier = new ImageMemoryBarrier { SType = StructureType.ImageMemoryBarrier, OldLayout = oldLayout, NewLayout = newLayout, SrcQueueFamilyIndex = Vk.QueueFamilyIgnored, DstQueueFamilyIndex = Vk.QueueFamilyIgnored, Image = image, SubresourceRange = new ImageSubresourceRange { AspectMask = ImageAspectFlags.ColorBit, BaseMipLevel = 0, LevelCount = 1, BaseArrayLayer = 0, LayerCount = 1 } }; context.Api.CmdPipelineBarrier( cmd, PipelineStageFlags.TopOfPipeBit, PipelineStageFlags.TransferBit, 0, 0, null, 0, null, 1, &barrier); } // unsafe private void TransitionImageLayout(CommandBuffer cmd, Silk.NET.Vulkan.Image image, ImageLayout oldLayout, ImageLayout newLayout) // { // var barrier = new ImageMemoryBarrier // { // SType = StructureType.ImageMemoryBarrier, // OldLayout = oldLayout, // NewLayout = newLayout, // SrcQueueFamilyIndex = 0, // DstQueueFamilyIndex = 0, // Image = image, // SubresourceRange = new ImageSubresourceRange // { // AspectMask = ImageAspectFlags.ColorBit, // BaseMipLevel = 0, // LevelCount = 1, // BaseArrayLayer = 0, // LayerCount = 1 // } // }; // PipelineStageFlags srcStage; // PipelineStageFlags dstStage; // if (oldLayout == ImageLayout.Undefined && newLayout == ImageLayout.TransferDstOptimal) // { // barrier.SrcAccessMask = 0; // barrier.DstAccessMask = AccessFlags.TransferWriteBit; // srcStage = PipelineStageFlags.TopOfPipeBit; // dstStage = PipelineStageFlags.TransferBit; // } // else if (oldLayout == ImageLayout.Undefined && newLayout == ImageLayout.ShaderReadOnlyOptimal) // { // barrier.SrcAccessMask = 0; // barrier.DstAccessMask = AccessFlags.ShaderReadBit; // srcStage = PipelineStageFlags.TopOfPipeBit; // dstStage = PipelineStageFlags.FragmentShaderBit; // } // else if (oldLayout == ImageLayout.TransferDstOptimal && newLayout == ImageLayout.ShaderReadOnlyOptimal) // { // barrier.SrcAccessMask = AccessFlags.TransferWriteBit; // barrier.DstAccessMask = AccessFlags.ShaderReadBit; // srcStage = PipelineStageFlags.TransferBit; // dstStage = PipelineStageFlags.FragmentShaderBit; // } // else // { // srcStage = PipelineStageFlags.AllCommandsBit; // dstStage = PipelineStageFlags.AllCommandsBit; // } // context!.Api.CmdPipelineBarrier(cmd, srcStage, dstStage, 0, 0, null, 0, null, 1, &barrier); // } } public partial class CameraView : Control { private bool _initialized = false; private Compositor? _compositor; private CompositionSurfaceVisual? _visual; private ICompositionGpuInterop? _gpuInterop; private CompositionDrawingSurface? Surface { get; set; } private bool _compositionRequested; private VkRenderer? _renerer; private VkImage? _latestImage; public CameraView() { } protected override void OnAttachedToVisualTree(VisualTreeAttachmentEventArgs e) { base.OnAttachedToVisualTree(e); Initialize(); } protected override void OnDetachedFromLogicalTree(LogicalTreeAttachmentEventArgs e) { if (_initialized) Surface?.Dispose(); _initialized = false; base.OnDetachedFromLogicalTree(e); } private async void Initialize() { try { var selfVisual = ElementComposition.GetElementVisual(this)!; _compositor = selfVisual.Compositor; Surface = _compositor.CreateDrawingSurface(); _visual = _compositor.CreateSurfaceVisual(); _visual.Surface = Surface; ElementComposition.SetElementChildVisual(this, _visual); _gpuInterop = await _compositor.TryGetCompositionGpuInterop(); _renerer = new VkRenderer(); _initialized = true; } catch (Exception e) { Console.WriteLine($"Error while initialization: \n{e.Message}\n{e.StackTrace}"); } } protected override void OnPropertyChanged(AvaloniaPropertyChangedEventArgs change) { if (_visual is not null && change.Property == BoundsProperty) { _visual.Size = new Vector(Bounds.Width, Bounds.Height); Update(); } if (change.Property == DataContextProperty) { (DataContext as CameraViewModel)?.Update += (_, ea) => Update(ea.FrameBuffer, ea.StreamConfiguration); } base.OnPropertyChanged(change); } private void RenderToSurface(VkImage image) { var (handle, properties, semaphore) = image.Export(); var imageRenderedSemaphore = _gpuInterop!.ImportSemaphore(_renerer!.ExportSemaphore(false)); imageRenderedSemaphore.ImportCompleted.ContinueWith(t => { Console.WriteLine($"imageRenderedSemaphore.ImportCompleted {t.Status}"); if (t.Exception is not null) { Console.WriteLine($"Error: \n{t.Exception.Message}\n{t.Exception.StackTrace} "); foreach (var e in t.Exception.InnerExceptions) { Console.WriteLine(e.Message); Console.WriteLine(e.StackTrace); } } }); var renderCompletedSemaphore = _gpuInterop!.ImportSemaphore(_renerer!.ExportSemaphore(true)); renderCompletedSemaphore.ImportCompleted.ContinueWith(t => { Console.WriteLine($"renderCompletedSemaphore.ImportCompleted {t.Status}"); if (t.Exception is not null) { Console.WriteLine($"Error: \n{t.Exception.Message}\n{t.Exception.StackTrace} "); foreach (var e in t.Exception.InnerExceptions) { Console.WriteLine(e.Message); Console.WriteLine(e.StackTrace); } } }); Console.WriteLine("ImportImage"); var importedImage = _gpuInterop!.ImportImage(handle, properties); importedImage.ImportCompleted.ContinueWith(t => { Console.WriteLine($"importedImage.ImportCompleted {t.Status}"); if (t.Exception is not null) { Console.WriteLine($"Error: \n{t.Exception.Message}\n{t.Exception.StackTrace} "); foreach (var e in t.Exception.InnerExceptions) { Console.WriteLine(e.Message); Console.WriteLine(e.StackTrace); } } }); Dispatcher.UIThread.Invoke(async () => { await Surface!.UpdateWithSemaphoresAsync(importedImage, renderCompletedSemaphore, imageRenderedSemaphore) .ContinueWith((t) => { Console.WriteLine($"UpdateWithSemaphoresAsync {t.Status}"); if (t.Exception is not null) { Console.WriteLine($"Error while surface update: \n{t.Exception.Message}\n{t.Exception.StackTrace} "); foreach (var e in t.Exception.InnerExceptions) { Console.WriteLine(e.Message); Console.WriteLine(e.StackTrace); } } }); } ); } private void OnComposition() { _compositionRequested = false; var image = _latestImage; if (image is null) return; RenderToSurface(image); } private void Update(FrameBuffer? frameBuffer = null, StreamConfiguration? streamConfiguration = null) { if (!_initialized || _compositor is null) return; if (frameBuffer is not null && streamConfiguration is not null && _renerer is not null) { var image = _renerer.ImportOrUpdate(frameBuffer.Planes.First().Fd.Get(), checked((int)streamConfiguration.Size.Width), checked((int)streamConfiguration.Size.Height), checked((int)streamConfiguration.Stride)); _latestImage = image; } if (_compositionRequested) return; _compositionRequested = true; _compositor?.RequestCompositionUpdate(OnComposition); } }