using System.Security.Cryptography; using Avalonia; using Avalonia.Controls; using Avalonia.LogicalTree; using Avalonia.Platform; using Avalonia.Rendering.Composition; using Avalonia.Threading; using GSS2.Vulkan; using Iot.Device.FtCommon; using LibCameraSharp; using Microsoft.Extensions.Logging.Abstractions; using Silk.NET.Core; using Silk.NET.Vulkan; using Silk.NET.Vulkan.Extensions.KHR; namespace GSS2.Test; interface IImage { public (IPlatformHandle handle, PlatformGraphicsExternalImageProperties properties, Silk.NET.Vulkan.Semaphore semaphore) Export(); } interface IRenderer { public abstract IImage ImportOrUpdate(int fd, int width, int height); } class VkImage { public int Fd { get; set; } public int Width { get; set; } public int Height { get; set; } public ulong MemorySize { get; set; } public Silk.NET.Vulkan.Semaphore Semaphore { get; set; } public (IPlatformHandle handle, PlatformGraphicsExternalImageProperties properties, Silk.NET.Vulkan.Semaphore semaphore) Export() { return ( new PlatformHandle(new IntPtr(Fd), KnownPlatformGraphicsExternalImageHandleTypes.VulkanOpaquePosixFileDescriptor), new PlatformGraphicsExternalImageProperties { Width = Width, Height = Height, Format = PlatformGraphicsExternalImageFormat.R8G8B8A8UNorm, MemorySize = MemorySize }, Semaphore ); } } class VkRenderer { readonly VulkanContext context = new VulkanContext("GSS2.Test", "CameraView"); public IPlatformHandle ExportSemaphore(bool renderFinished) { int fd = context.ExportSemaphoreFd(renderFinished ? context.RenderFinishedSemaphore : context.ImageAvailableSemaphore); return new PlatformHandle(new IntPtr(fd), KnownPlatformGraphicsExternalImageHandleTypes.VulkanOpaquePosixFileDescriptor); } unsafe public VkImage ImportOrUpdate(int fd, int width, int height) { try { var extImageInfo = new ExternalMemoryImageCreateInfo { SType = StructureType.ExternalMemoryImageCreateInfo, HandleTypes = ExternalMemoryHandleTypeFlags.OpaqueFDBit }; var imageInfo = new ImageCreateInfo { SType = StructureType.ImageCreateInfo, PNext = &extImageInfo, ImageType = ImageType.Type2D, Format = Format.R8G8B8A8Unorm, Extent = new Extent3D { Width = checked((uint)width), Height = checked((uint)height), Depth = 1 }, MipLevels = 1, ArrayLayers = 1, Samples = SampleCountFlags.Count1Bit, Tiling = ImageTiling.Optimal, Usage = ImageUsageFlags.TransferDstBit | ImageUsageFlags.SampledBit, SharingMode = SharingMode.Exclusive, InitialLayout = ImageLayout.Undefined }; context.Api.CreateImage(context.Device, &imageInfo, null, out var image).ThrowOnError(); context.Api.GetImageMemoryRequirements(context.Device, image, out var memReq); var exportAllocInfo = new ExportMemoryAllocateInfo { SType = StructureType.ExportMemoryAllocateInfo, HandleTypes = ExternalMemoryHandleTypeFlags.OpaqueFDBit }; var allocInfo = new MemoryAllocateInfo { SType = StructureType.MemoryAllocateInfo, PNext = &exportAllocInfo, AllocationSize = memReq.Size, MemoryTypeIndex = checked((uint)FindMemoryType(memReq.MemoryTypeBits, MemoryPropertyFlags.DeviceLocalBit)) }; context.Api.AllocateMemory(context.Device, &allocInfo, null, out var memory).ThrowOnError(); context.Api.BindImageMemory(context.Device, image, memory, 0).ThrowOnError(); var poolInfo = new CommandPoolCreateInfo { SType = StructureType.CommandPoolCreateInfo, Flags = CommandPoolCreateFlags.ResetCommandBufferBit, QueueFamilyIndex = context.QueueFamilyIndex }; context.Api.CreateCommandPool(context.Device, &poolInfo, null, out var commandPool).ThrowOnError(); var cmdBufferAllocInfo = new CommandBufferAllocateInfo { SType = StructureType.CommandBufferAllocateInfo, CommandPool = commandPool, Level = CommandBufferLevel.Primary, CommandBufferCount = 1 }; context.Api.AllocateCommandBuffers(context.Device, &cmdBufferAllocInfo, out var cmd).ThrowOnError(); var clearColor = new ClearColorValue((float)Random.Shared.NextDouble(), (float)Random.Shared.NextDouble(), (float)Random.Shared.NextDouble(), (float)Random.Shared.NextDouble()); var range = new ImageSubresourceRange { AspectMask = ImageAspectFlags.ColorBit, BaseMipLevel = 0, LevelCount = 1, BaseArrayLayer = 0, LayerCount = 1 }; var beginInfo = new CommandBufferBeginInfo { SType = StructureType.CommandBufferBeginInfo, Flags = CommandBufferUsageFlags.OneTimeSubmitBit }; context.Api.BeginCommandBuffer(cmd, &beginInfo).ThrowOnError(); TransitionImageLayout(cmd, image, ImageLayout.Undefined, ImageLayout.TransferDstOptimal); context.Api.CmdClearColorImage(cmd, image, ImageLayout.TransferDstOptimal, &clearColor, 1, &range); TransitionImageLayout(cmd, image, ImageLayout.TransferDstOptimal, ImageLayout.General); context.Api.EndCommandBuffer(cmd).ThrowOnError(); var renderFinished = context.RenderFinishedSemaphore; var submitInfo = new SubmitInfo { SType = StructureType.SubmitInfo, CommandBufferCount = 1, PCommandBuffers = &cmd, PSignalSemaphores = &renderFinished, SignalSemaphoreCount = 1 }; context.Api.QueueSubmit(context.Queue, 1, &submitInfo, default).ThrowOnError(); var getFdInfo = new MemoryGetFdInfoKHR { SType = StructureType.MemoryGetFDInfoKhr, Memory = memory, HandleType = ExternalMemoryHandleTypeFlags.OpaqueFDBit }; if (context.Api.TryGetDeviceExtension(context.Instance, context.Device, out var khrExternalMemoryFd)) Console.WriteLine("KhrExternalMemoryFd import success"); else throw new Exception("KhrExternalMemoryFd import error"); khrExternalMemoryFd.GetMemoryF(context.Device, &getFdInfo, out var fd1).ThrowOnError(); Console.WriteLine($"Image export success {fd1}"); return new VkImage() { Width = width, Height = height, Fd = fd1, MemorySize = memReq.Size }; } catch (Exception e) { Console.WriteLine(e.Message); Console.WriteLine(e.StackTrace); throw; } } private int FindMemoryType(uint typeFilter, MemoryPropertyFlags properties) { context!.Api.GetPhysicalDeviceMemoryProperties(context.PhysicalDevice, out var memProps); for (int i = 0; i < memProps.MemoryTypeCount; i++) if ((typeFilter & (1 << i)) != 0 && (memProps.MemoryTypes[i].PropertyFlags & properties) == properties) return i; return -1; } unsafe private void TransitionImageLayout(CommandBuffer cmd, Silk.NET.Vulkan.Image image, ImageLayout oldLayout, ImageLayout newLayout) { var barrier = new ImageMemoryBarrier { SType = StructureType.ImageMemoryBarrier, OldLayout = oldLayout, NewLayout = newLayout, SrcQueueFamilyIndex = 0, DstQueueFamilyIndex = 0, Image = image, SubresourceRange = new ImageSubresourceRange { AspectMask = ImageAspectFlags.ColorBit, BaseMipLevel = 0, LevelCount = 1, BaseArrayLayer = 0, LayerCount = 1 } }; PipelineStageFlags srcStage; PipelineStageFlags dstStage; if (oldLayout == ImageLayout.Undefined && newLayout == ImageLayout.TransferDstOptimal) { barrier.SrcAccessMask = 0; barrier.DstAccessMask = AccessFlags.TransferWriteBit; srcStage = PipelineStageFlags.TopOfPipeBit; dstStage = PipelineStageFlags.TransferBit; } else if (oldLayout == ImageLayout.TransferDstOptimal && newLayout == ImageLayout.ShaderReadOnlyOptimal) { barrier.SrcAccessMask = AccessFlags.TransferWriteBit; barrier.DstAccessMask = AccessFlags.ShaderReadBit; srcStage = PipelineStageFlags.TransferBit; dstStage = PipelineStageFlags.FragmentShaderBit; } else { srcStage = PipelineStageFlags.AllCommandsBit; dstStage = PipelineStageFlags.AllCommandsBit; } context!.Api.CmdPipelineBarrier(cmd, srcStage, dstStage, 0, 0, null, 0, null, 1, &barrier); } } public partial class CameraView : Control { private bool _initialized = false; private Compositor? _compositor; private CompositionSurfaceVisual? _visual; private ICompositionGpuInterop? _gpuInterop; private CompositionDrawingSurface? Surface { get; set; } private bool _compositionRequested; private VkRenderer? _renerer; private VkImage? _latestImage; public CameraView() { } protected override void OnAttachedToVisualTree(VisualTreeAttachmentEventArgs e) { base.OnAttachedToVisualTree(e); Initialize(); } protected override void OnDetachedFromLogicalTree(LogicalTreeAttachmentEventArgs e) { if (_initialized) Surface?.Dispose(); _initialized = false; base.OnDetachedFromLogicalTree(e); } private async void Initialize() { try { var selfVisual = ElementComposition.GetElementVisual(this)!; _compositor = selfVisual.Compositor; Surface = _compositor.CreateDrawingSurface(); _visual = _compositor.CreateSurfaceVisual(); _visual.Surface = Surface; ElementComposition.SetElementChildVisual(this, _visual); _gpuInterop = await _compositor.TryGetCompositionGpuInterop(); _renerer = new VkRenderer(); _initialized = true; } catch (Exception e) { Console.WriteLine($"Error while initialization: \n{e.Message}\n{e.StackTrace}"); } } protected override void OnPropertyChanged(AvaloniaPropertyChangedEventArgs change) { if (_visual is not null && change.Property == BoundsProperty) { _visual.Size = new Vector(Bounds.Width, Bounds.Height); Update(); } if (change.Property == DataContextProperty) { (DataContext as CameraViewModel)?.Update += (_, ea) => Update(ea.FrameBuffer, ea.StreamConfiguration); } base.OnPropertyChanged(change); } private void RenderToSurface(VkImage image) { var (handle, properties, semaphore) = image.Export(); var imageRenderedSemaphore = _gpuInterop!.ImportSemaphore(_renerer!.ExportSemaphore(false)); var renderCompletedSemaphore = _gpuInterop!.ImportSemaphore(_renerer!.ExportSemaphore(true)); var importedImage = _gpuInterop!.ImportImage(handle, properties); Dispatcher.UIThread.Invoke(async () => { await importedImage.ImportCompleted; await Surface!.UpdateWithSemaphoresAsync(importedImage, renderCompletedSemaphore, imageRenderedSemaphore) .ContinueWith((t) => { if (t.Exception is not null) { Console.WriteLine($"Error while surface update: \n{t.Exception.Message}\n{t.Exception.StackTrace} "); foreach (var e in t.Exception.InnerExceptions) { Console.WriteLine(e.Message); Console.WriteLine(e.StackTrace); } } }); } ); } private void OnComposition() { _compositionRequested = false; var image = _latestImage; if (image is null) return; RenderToSurface(image); } private void Update(FrameBuffer? frameBuffer = null, StreamConfiguration? streamConfiguration = null) { if (!_initialized || _compositor is null) return; if (frameBuffer is not null && streamConfiguration is not null && _renerer is not null) { var image = _renerer.ImportOrUpdate(frameBuffer.Planes.First().Fd.Get(), checked((int)streamConfiguration.Size.Width), checked((int)streamConfiguration.Size.Height)); _latestImage = image; } if (_compositionRequested) return; _compositionRequested = true; _compositor?.RequestCompositionUpdate(OnComposition); } }