diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 04ebe1575b798..3d33f51e5d590 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -35316,6 +35316,10 @@ ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/vertex_descriptor_vk.h ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/vk.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/vma.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/vma.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/yuv_conversion_library_vk.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/yuv_conversion_library_vk.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/yuv_conversion_vk.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/yuv_conversion_vk.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/renderer/blit_command.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/renderer/blit_command.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/renderer/blit_pass.cc + ../../../flutter/LICENSE @@ -38163,6 +38167,10 @@ FILE: ../../../flutter/impeller/renderer/backend/vulkan/vertex_descriptor_vk.h FILE: ../../../flutter/impeller/renderer/backend/vulkan/vk.h FILE: ../../../flutter/impeller/renderer/backend/vulkan/vma.cc FILE: ../../../flutter/impeller/renderer/backend/vulkan/vma.h +FILE: ../../../flutter/impeller/renderer/backend/vulkan/yuv_conversion_library_vk.cc +FILE: ../../../flutter/impeller/renderer/backend/vulkan/yuv_conversion_library_vk.h +FILE: ../../../flutter/impeller/renderer/backend/vulkan/yuv_conversion_vk.cc +FILE: ../../../flutter/impeller/renderer/backend/vulkan/yuv_conversion_vk.h FILE: ../../../flutter/impeller/renderer/blit_command.cc FILE: ../../../flutter/impeller/renderer/blit_command.h FILE: ../../../flutter/impeller/renderer/blit_pass.cc diff --git a/impeller/renderer/backend/vulkan/BUILD.gn b/impeller/renderer/backend/vulkan/BUILD.gn index ac457094136bf..952ae0cb26b5f 100644 --- a/impeller/renderer/backend/vulkan/BUILD.gn +++ b/impeller/renderer/backend/vulkan/BUILD.gn @@ -115,6 +115,10 @@ impeller_component("vulkan") { "vk.h", "vma.cc", "vma.h", + "yuv_conversion_library_vk.cc", + "yuv_conversion_library_vk.h", + "yuv_conversion_vk.cc", + "yuv_conversion_vk.h", ] public_deps = [ diff --git a/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.cc b/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.cc index 1f7619c6d95ae..0ec9e2725ff02 100644 --- a/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.cc +++ b/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.cc @@ -6,45 +6,94 @@ #include +#include "impeller/renderer/backend/vulkan/context_vk.h" #include "impeller/renderer/backend/vulkan/texture_source_vk.h" +#include "impeller/renderer/backend/vulkan/yuv_conversion_library_vk.h" #ifdef FML_OS_ANDROID namespace impeller { -namespace { +using AHBProperties = vk::StructureChain< + // For VK_ANDROID_external_memory_android_hardware_buffer + vk::AndroidHardwareBufferPropertiesANDROID, + // For VK_ANDROID_external_memory_android_hardware_buffer + vk::AndroidHardwareBufferFormatPropertiesANDROID>; -bool GetHardwareBufferProperties( +static vk::UniqueImage CreateVKImageWrapperForAndroidHarwareBuffer( const vk::Device& device, - struct AHardwareBuffer* hardware_buffer, - ::impeller::vk::AndroidHardwareBufferPropertiesANDROID* ahb_props, - ::impeller::vk::AndroidHardwareBufferFormatPropertiesANDROID* - ahb_format_props) { - FML_CHECK(ahb_format_props != nullptr); - FML_CHECK(ahb_props != nullptr); - ahb_props->pNext = ahb_format_props; - ::impeller::vk::Result result = - device.getAndroidHardwareBufferPropertiesANDROID(hardware_buffer, - ahb_props); - if (result != impeller::vk::Result::eSuccess) { - return false; - } - return true; -} + const AHBProperties& ahb_props, + const AHardwareBuffer_Desc& ahb_desc) { + const auto& ahb_format = + ahb_props.get(); + + vk::StructureChain + image_chain; + + auto& image_info = image_chain.get(); + + vk::ImageUsageFlags image_usage_flags; + if (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE) { + image_usage_flags |= vk::ImageUsageFlagBits::eSampled; + } + if (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_FRAMEBUFFER) { + image_usage_flags |= vk::ImageUsageFlagBits::eColorAttachment; + } + + vk::ImageCreateFlags image_create_flags; + if (ahb_desc.usage & AHARDWAREBUFFER_USAGE_PROTECTED_CONTENT) { + image_create_flags |= vk::ImageCreateFlagBits::eProtected; + } + if (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_CUBE_MAP) { + image_create_flags |= vk::ImageCreateFlagBits::eCubeCompatible; + } -vk::ExternalFormatANDROID MakeExternalFormat( - const vk::AndroidHardwareBufferFormatPropertiesANDROID& format_props) { - vk::ExternalFormatANDROID external_format; - external_format.pNext = nullptr; - external_format.externalFormat = 0; - if (format_props.format == vk::Format::eUndefined) { - external_format.externalFormat = format_props.externalFormat; + image_info.imageType = vk::ImageType::e2D; + image_info.format = ahb_format.format; + image_info.extent.width = ahb_desc.width; + image_info.extent.height = ahb_desc.height; + image_info.extent.depth = 1; + image_info.mipLevels = + (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_MIPMAP_COMPLETE) + ? ISize{ahb_desc.width, ahb_desc.height}.MipCount() + : 1u; + image_info.arrayLayers = ahb_desc.layers; + image_info.samples = vk::SampleCountFlagBits::e1; + image_info.tiling = vk::ImageTiling::eOptimal; + image_info.usage = image_usage_flags; + image_info.flags = image_create_flags; + image_info.sharingMode = vk::SharingMode::eExclusive; + image_info.initialLayout = vk::ImageLayout::eUndefined; + + image_chain.get().handleTypes = + vk::ExternalMemoryHandleTypeFlagBits::eAndroidHardwareBufferANDROID; + + // If the format isn't natively supported by Vulkan (i.e, be a part of the + // base vkFormat enum), an untyped "external format" must be specified when + // creating the image and the image views. Usually includes YUV formats. + if (ahb_format.format == vk::Format::eUndefined) { + image_chain.get().externalFormat = + ahb_format.externalFormat; + } else { + image_chain.unlink(); } - return external_format; + + auto image = device.createImageUnique(image_chain.get()); + if (image.result != vk::Result::eSuccess) { + VALIDATION_LOG << "Could not create image for external buffer: " + << vk::to_string(image.result); + return {}; + } + + return std::move(image.value); } // Returns -1 if not found. -int FindMemoryTypeIndex( +static int FindMemoryTypeIndex( const vk::AndroidHardwareBufferPropertiesANDROID& props) { uint32_t memory_type_bits = props.memoryTypeBits; int32_t type_index = -1; @@ -58,132 +107,274 @@ int FindMemoryTypeIndex( return type_index; } -} // namespace - -AndroidHardwareBufferTextureSourceVK::AndroidHardwareBufferTextureSourceVK( - TextureDescriptor desc, +static vk::UniqueDeviceMemory ImportVKDeviceMemoryFromAndroidHarwareBuffer( const vk::Device& device, + const vk::Image& image, struct AHardwareBuffer* hardware_buffer, - const AHardwareBuffer_Desc& hardware_buffer_desc) - : TextureSourceVK(desc), device_(device) { - vk::AndroidHardwareBufferFormatPropertiesANDROID ahb_format_props; - vk::AndroidHardwareBufferPropertiesANDROID ahb_props; - if (!GetHardwareBufferProperties(device, hardware_buffer, &ahb_props, - &ahb_format_props)) { - return; - } - vk::ExternalFormatANDROID external_format = - MakeExternalFormat(ahb_format_props); - vk::ExternalMemoryImageCreateInfo external_memory_image_info; - external_memory_image_info.pNext = &external_format; - external_memory_image_info.handleTypes = - vk::ExternalMemoryHandleTypeFlagBits::eAndroidHardwareBufferANDROID; - const int memory_type_index = FindMemoryTypeIndex(ahb_props); + const AHBProperties& ahb_props) { + const int memory_type_index = FindMemoryTypeIndex(ahb_props.get()); if (memory_type_index < 0) { - FML_LOG(ERROR) << "Could not find memory type."; - return; + VALIDATION_LOG << "Could not find memory type of external image."; + return {}; } - vk::ImageCreateFlags image_create_flags; - vk::ImageUsageFlags image_usage_flags; - if (hardware_buffer_desc.usage & AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE) { - image_usage_flags |= impeller::vk::ImageUsageFlagBits::eSampled | - impeller::vk::ImageUsageFlagBits::eInputAttachment; - } - if (hardware_buffer_desc.usage & AHARDWAREBUFFER_USAGE_GPU_COLOR_OUTPUT) { - image_usage_flags |= impeller::vk::ImageUsageFlagBits::eColorAttachment; - } - if (hardware_buffer_desc.usage & AHARDWAREBUFFER_USAGE_PROTECTED_CONTENT) { - image_create_flags |= impeller::vk::ImageCreateFlagBits::eProtected; - } - - vk::ImageCreateInfo image_create_info; - image_create_info.pNext = &external_memory_image_info; - image_create_info.imageType = vk::ImageType::e2D; - image_create_info.format = ahb_format_props.format; - image_create_info.extent.width = hardware_buffer_desc.width; - image_create_info.extent.height = hardware_buffer_desc.height; - image_create_info.extent.depth = 1; - image_create_info.mipLevels = 1; - image_create_info.arrayLayers = 1; - image_create_info.samples = vk::SampleCountFlagBits::e1; - image_create_info.tiling = vk::ImageTiling::eOptimal; - image_create_info.usage = image_usage_flags; - image_create_info.flags = image_create_flags; - image_create_info.sharingMode = vk::SharingMode::eExclusive; - image_create_info.initialLayout = vk::ImageLayout::eUndefined; - - vk::ResultValue maybe_image = - device.createImage(image_create_info); - if (maybe_image.result != vk::Result::eSuccess) { - FML_LOG(ERROR) << "device.createImage failed: " - << static_cast(maybe_image.result); - return; - } - vk::Image image = maybe_image.value; + vk::StructureChain + memory_chain; - vk::ImportAndroidHardwareBufferInfoANDROID ahb_import_info; - ahb_import_info.pNext = nullptr; - ahb_import_info.buffer = hardware_buffer; + auto& mem_alloc_info = memory_chain.get(); + mem_alloc_info.allocationSize = ahb_props.get().allocationSize; + mem_alloc_info.memoryTypeIndex = memory_type_index; - vk::MemoryDedicatedAllocateInfo dedicated_alloc_info; - dedicated_alloc_info.pNext = &ahb_import_info; + auto& dedicated_alloc_info = + memory_chain.get(); dedicated_alloc_info.image = image; - dedicated_alloc_info.buffer = VK_NULL_HANDLE; - vk::MemoryAllocateInfo mem_alloc_info; - mem_alloc_info.pNext = &dedicated_alloc_info; - mem_alloc_info.allocationSize = ahb_props.allocationSize; - mem_alloc_info.memoryTypeIndex = memory_type_index; + auto& ahb_import_info = + memory_chain.get(); + ahb_import_info.buffer = hardware_buffer; - vk::ResultValue allocate_result = - device.allocateMemory(mem_alloc_info); - if (allocate_result.result != vk::Result::eSuccess) { - FML_LOG(ERROR) << "vkAllocateMemory failed : " - << static_cast(allocate_result.result); - device.destroyImage(image); - return; + auto device_memory = device.allocateMemoryUnique(memory_chain.get()); + if (device_memory.result != vk::Result::eSuccess) { + VALIDATION_LOG << "Could not allocate device memory for external image : " + << vk::to_string(device_memory.result); + return {}; } - vk::DeviceMemory device_memory = allocate_result.value; - // Bind memory to the image object. - vk::Result bind_image_result = - device.bindImageMemory(image, device_memory, 0); - if (bind_image_result != vk::Result::eSuccess) { - FML_LOG(ERROR) << "vkBindImageMemory failed : " - << static_cast(bind_image_result); - device.destroyImage(image); - device.freeMemory(device_memory); - return; + return std::move(device_memory.value); +} + +static std::shared_ptr CreateYUVConversion( + const ContextVK& context, + const AHBProperties& ahb_props) { + YUVConversionDescriptorVK conversion_chain; + + const auto& ahb_format = + ahb_props.get(); + + auto& conversion_info = conversion_chain.get(); + + conversion_info.format = ahb_format.format; + conversion_info.ycbcrModel = ahb_format.suggestedYcbcrModel; + conversion_info.ycbcrRange = ahb_format.suggestedYcbcrRange; + conversion_info.components = ahb_format.samplerYcbcrConversionComponents; + conversion_info.xChromaOffset = ahb_format.suggestedXChromaOffset; + conversion_info.yChromaOffset = ahb_format.suggestedYChromaOffset; + // If the potential format features of the sampler Y′CBCR conversion do not + // support VK_FORMAT_FEATURE_SAMPLED_IMAGE_YCBCR_CONVERSION_LINEAR_FILTER_BIT, + // chromaFilter must not be VK_FILTER_LINEAR. + // + // Since we are not checking, let's just default to a safe value. + conversion_info.chromaFilter = vk::Filter::eNearest; + conversion_info.forceExplicitReconstruction = false; + + if (conversion_info.format == vk::Format::eUndefined) { + auto& external_format = conversion_chain.get(); + external_format.externalFormat = ahb_format.externalFormat; + } else { + conversion_chain.unlink(); } - image_ = image; - device_memory_ = device_memory; - // Create image view. - vk::ImageViewCreateInfo view_info; - view_info.image = image_; + return context.GetYUVConversionLibrary()->GetConversion(conversion_chain); +} + +static vk::UniqueImageView CreateVKImageView( + const vk::Device& device, + const vk::Image& image, + const vk::SamplerYcbcrConversion& yuv_conversion, + const AHBProperties& ahb_props, + const AHardwareBuffer_Desc& ahb_desc) { + const auto& ahb_format = + ahb_props.get(); + + vk::StructureChain + view_chain; + + auto& view_info = view_chain.get(); + + view_info.image = image; view_info.viewType = vk::ImageViewType::e2D; - view_info.format = ToVKImageFormat(desc.format); + view_info.format = ahb_format.format; view_info.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; view_info.subresourceRange.baseMipLevel = 0u; view_info.subresourceRange.baseArrayLayer = 0u; - view_info.subresourceRange.levelCount = desc.mip_count; - view_info.subresourceRange.layerCount = ToArrayLayerCount(desc.type); - auto [view_result, view] = device.createImageViewUnique(view_info); - if (view_result != vk::Result::eSuccess) { - FML_LOG(ERROR) << "createImageViewUnique failed : " - << static_cast(view_result); + view_info.subresourceRange.levelCount = + (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_MIPMAP_COMPLETE) + ? ISize{ahb_desc.width, ahb_desc.height}.MipCount() + : 1u; + view_info.subresourceRange.layerCount = ahb_desc.layers; + + // We need a custom YUV conversion only if we don't recognize the format. + if (view_info.format == vk::Format::eUndefined) { + view_chain.get().conversion = + yuv_conversion; + } else { + view_chain.unlink(); + } + + auto image_view = device.createImageViewUnique(view_info); + if (image_view.result != vk::Result::eSuccess) { + VALIDATION_LOG << "Could not create external image view: " + << vk::to_string(image_view.result); + return {}; + } + + return std::move(image_view.value); +} + +static PixelFormat ToPixelFormat(AHardwareBuffer_Format format) { + switch (format) { + case AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM: + return PixelFormat::kR8G8B8A8UNormInt; + case AHARDWAREBUFFER_FORMAT_R16G16B16A16_FLOAT: + return PixelFormat::kR16G16B16A16Float; + case AHARDWAREBUFFER_FORMAT_D24_UNORM_S8_UINT: + return PixelFormat::kD24UnormS8Uint; + case AHARDWAREBUFFER_FORMAT_D32_FLOAT_S8_UINT: + return PixelFormat::kD32FloatS8UInt; + case AHARDWAREBUFFER_FORMAT_S8_UINT: + return PixelFormat::kS8UInt; + case AHARDWAREBUFFER_FORMAT_R8_UNORM: + return PixelFormat::kR8UNormInt; + case AHARDWAREBUFFER_FORMAT_R10G10B10A10_UNORM: + case AHARDWAREBUFFER_FORMAT_R16G16_UINT: + case AHARDWAREBUFFER_FORMAT_D32_FLOAT: + case AHARDWAREBUFFER_FORMAT_R16_UINT: + case AHARDWAREBUFFER_FORMAT_D24_UNORM: + case AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420: + case AHARDWAREBUFFER_FORMAT_YCbCr_P010: + case AHARDWAREBUFFER_FORMAT_BLOB: + case AHARDWAREBUFFER_FORMAT_R10G10B10A2_UNORM: + case AHARDWAREBUFFER_FORMAT_R5G6B5_UNORM: + case AHARDWAREBUFFER_FORMAT_R8G8B8_UNORM: + case AHARDWAREBUFFER_FORMAT_D16_UNORM: + case AHARDWAREBUFFER_FORMAT_R8G8B8X8_UNORM: + // Not understood by the rest of Impeller. Use a placeholder but create + // the native image and image views using the right external format. + break; + } + return PixelFormat::kR8G8B8A8UNormInt; +} + +static TextureType ToTextureType(const AHardwareBuffer_Desc& ahb_desc) { + if (ahb_desc.layers == 1u) { + return TextureType::kTexture2D; + } + if (ahb_desc.layers % 6u == 0 && + (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_CUBE_MAP)) { + return TextureType::kTextureCube; + } + // Our texture types seem to understand external OES textures. Should these be + // wired up instead? + return TextureType::kTexture2D; +} + +static TextureDescriptor ToTextureDescriptor( + const AHardwareBuffer_Desc& ahb_desc) { + const auto ahb_size = ISize{ahb_desc.width, ahb_desc.height}; + TextureDescriptor desc; + // We are not going to touch hardware buffers on the CPU or use them as + // transient attachments. Just treat them as device private. + desc.storage_mode = StorageMode::kDevicePrivate; + desc.format = + ToPixelFormat(static_cast(ahb_desc.format)); + desc.size = ahb_size; + desc.type = ToTextureType(ahb_desc); + desc.sample_count = SampleCount::kCount1; + desc.compression_type = CompressionType::kLossless; + desc.mip_count = (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_MIPMAP_COMPLETE) + ? ahb_size.MipCount() + : 1u; + return desc; +} + +AndroidHardwareBufferTextureSourceVK::AndroidHardwareBufferTextureSourceVK( + const std::shared_ptr& context, + struct AHardwareBuffer* ahb, + const AHardwareBuffer_Desc& ahb_desc) + : TextureSourceVK(ToTextureDescriptor(ahb_desc)) { + if (!context) { + VALIDATION_LOG << "Invalid context."; + return; + } + + const auto& device = context->GetDevice(); + + AHBProperties ahb_props; + + if (device.getAndroidHardwareBufferPropertiesANDROID(ahb, &ahb_props.get()) != + vk::Result::eSuccess) { + VALIDATION_LOG << "Could not determine properties of the Android hardware " + "buffer."; return; } - image_view_ = std::move(view); + + const auto& ahb_format = + ahb_props.get(); + + // Create an image to refer to our external image. + auto image = + CreateVKImageWrapperForAndroidHarwareBuffer(device, ahb_props, ahb_desc); + if (!image) { + return; + } + + // Create a device memory allocation to refer to our external image. + auto device_memory = ImportVKDeviceMemoryFromAndroidHarwareBuffer( + device, image.get(), ahb, ahb_props); + if (!device_memory) { + return; + } + + // Bind the image to the image memory. + if (auto result = device.bindImageMemory(image.get(), device_memory.get(), 0); + result != vk::Result::eSuccess) { + VALIDATION_LOG << "Could not bind external device memory to image : " + << vk::to_string(result); + return; + } + + // Figure out how to perform YUV conversions. + auto yuv_conversion = CreateYUVConversion(*context, ahb_props); + if (!yuv_conversion || !yuv_conversion->IsValid()) { + return; + } + + // Create image view for the newly created image. + auto image_view = CreateVKImageView(device, // + image.get(), // + yuv_conversion->GetConversion(), // + ahb_props, // + ahb_desc // + ); + if (!image_view) { + return; + } + + needs_yuv_conversion_ = ahb_format.format == vk::Format::eUndefined; + device_memory_ = std::move(device_memory); + image_ = std::move(image); + yuv_conversion_ = std::move(yuv_conversion); + image_view_ = std::move(image_view); + +#ifdef IMPELLER_DEBUG + context->SetDebugName(device_memory_.get(), "AHB Device Memory"); + context->SetDebugName(image_.get(), "AHB Image"); + context->SetDebugName(yuv_conversion_->GetConversion(), "AHB YUV Conversion"); + context->SetDebugName(image_view_.get(), "AHB ImageView"); +#endif // IMPELLER_DEBUG + is_valid_ = true; } // |TextureSourceVK| -AndroidHardwareBufferTextureSourceVK::~AndroidHardwareBufferTextureSourceVK() { - device_.destroyImage(image_); - device_.freeMemory(device_memory_); -} +AndroidHardwareBufferTextureSourceVK::~AndroidHardwareBufferTextureSourceVK() = + default; bool AndroidHardwareBufferTextureSourceVK::IsValid() const { return is_valid_; @@ -191,23 +382,31 @@ bool AndroidHardwareBufferTextureSourceVK::IsValid() const { // |TextureSourceVK| vk::Image AndroidHardwareBufferTextureSourceVK::GetImage() const { - FML_CHECK(IsValid()); - return image_; + return image_.get(); } // |TextureSourceVK| vk::ImageView AndroidHardwareBufferTextureSourceVK::GetImageView() const { - FML_CHECK(IsValid()); return image_view_.get(); } // |TextureSourceVK| vk::ImageView AndroidHardwareBufferTextureSourceVK::GetRenderTargetView() const { - FML_CHECK(IsValid()); return image_view_.get(); } +// |TextureSourceVK| +bool AndroidHardwareBufferTextureSourceVK::IsSwapchainImage() const { + return false; +} + +// |TextureSourceVK| +std::shared_ptr +AndroidHardwareBufferTextureSourceVK::GetYUVConversion() const { + return needs_yuv_conversion_ ? yuv_conversion_ : nullptr; +} + } // namespace impeller #endif diff --git a/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.h b/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.h index 19cfab1a2b765..724308cb2b6fb 100644 --- a/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.h +++ b/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.h @@ -15,17 +15,31 @@ #include "impeller/renderer/backend/vulkan/formats_vk.h" #include "impeller/renderer/backend/vulkan/texture_source_vk.h" #include "impeller/renderer/backend/vulkan/vk.h" +#include "impeller/renderer/backend/vulkan/yuv_conversion_vk.h" #include #include namespace impeller { +class ContextVK; + +//------------------------------------------------------------------------------ +/// @brief A texture source that wraps an instance of AHardwareBuffer. +/// +/// The formats and conversions supported by Android Hardware +/// Buffers are a superset of those supported by Impeller (and +/// Vulkan for that matter). Impeller and Vulkan descriptors +/// obtained from the these texture sources are advisory and it +/// usually isn't possible to create copies of images and image +/// views held in these texture sources using the inferred +/// descriptors. The objects are meant to be used directly (either +/// as render targets or sources for sampling), not copied. +/// class AndroidHardwareBufferTextureSourceVK final : public TextureSourceVK { public: AndroidHardwareBufferTextureSourceVK( - TextureDescriptor desc, - const vk::Device& device, + const std::shared_ptr& context, struct AHardwareBuffer* hardware_buffer, const AHardwareBuffer_Desc& hardware_buffer_desc); @@ -43,14 +57,18 @@ class AndroidHardwareBufferTextureSourceVK final : public TextureSourceVK { bool IsValid() const; - bool IsSwapchainImage() const override { return false; } + // |TextureSourceVK| + bool IsSwapchainImage() const override; - private: - const vk::Device& device_; - vk::Image image_ = VK_NULL_HANDLE; - vk::UniqueImageView image_view_ = {}; - vk::DeviceMemory device_memory_ = VK_NULL_HANDLE; + // |TextureSourceVK| + std::shared_ptr GetYUVConversion() const override; + private: + vk::UniqueDeviceMemory device_memory_; + vk::UniqueImage image_; + vk::UniqueImageView image_view_; + std::shared_ptr yuv_conversion_; + bool needs_yuv_conversion_ = false; bool is_valid_ = false; AndroidHardwareBufferTextureSourceVK( diff --git a/impeller/renderer/backend/vulkan/capabilities_vk.cc b/impeller/renderer/backend/vulkan/capabilities_vk.cc index ab5214747b5b5..b5ad0aba23f93 100644 --- a/impeller/renderer/backend/vulkan/capabilities_vk.cc +++ b/impeller/renderer/backend/vulkan/capabilities_vk.cc @@ -9,7 +9,6 @@ #include "impeller/base/validation.h" #include "impeller/core/formats.h" #include "impeller/renderer/backend/vulkan/vk.h" -#include "vulkan/vulkan_core.h" namespace impeller { @@ -153,25 +152,58 @@ CapabilitiesVK::GetEnabledInstanceExtensions() const { return required; } -static const char* GetDeviceExtensionName(OptionalDeviceExtensionVK ext) { +static const char* GetExtensionName(RequiredCommonDeviceExtensionVK ext) { + switch (ext) { + case RequiredCommonDeviceExtensionVK::kKHRSwapchain: + return VK_KHR_SWAPCHAIN_EXTENSION_NAME; + case RequiredCommonDeviceExtensionVK::kLast: + return "Unknown"; + } + FML_UNREACHABLE(); +} + +static const char* GetExtensionName(RequiredAndroidDeviceExtensionVK ext) { + switch (ext) { + case RequiredAndroidDeviceExtensionVK:: + kANDROIDExternalMemoryAndroidHardwareBuffer: + return "VK_ANDROID_external_memory_android_hardware_buffer"; + case RequiredAndroidDeviceExtensionVK::kKHRSamplerYcbcrConversion: + return VK_KHR_SAMPLER_YCBCR_CONVERSION_EXTENSION_NAME; + case RequiredAndroidDeviceExtensionVK::kKHRExternalMemory: + return VK_KHR_EXTERNAL_MEMORY_EXTENSION_NAME; + case RequiredAndroidDeviceExtensionVK::kEXTQueueFamilyForeign: + return VK_EXT_QUEUE_FAMILY_FOREIGN_EXTENSION_NAME; + case RequiredAndroidDeviceExtensionVK::kKHRDedicatedAllocation: + return VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME; + case RequiredAndroidDeviceExtensionVK::kLast: + return "Unknown"; + } + FML_UNREACHABLE(); +} + +static const char* GetExtensionName(OptionalDeviceExtensionVK ext) { switch (ext) { case OptionalDeviceExtensionVK::kEXTPipelineCreationFeedback: return VK_EXT_PIPELINE_CREATION_FEEDBACK_EXTENSION_NAME; + case OptionalDeviceExtensionVK::kVKKHRPortabilitySubset: + return "VK_KHR_portability_subset"; case OptionalDeviceExtensionVK::kLast: return "Unknown"; } - return "Unknown"; + FML_UNREACHABLE(); } -static void IterateOptionalDeviceExtensions( - const std::function& it) { +template +static bool IterateExtensions(const std::function& it) { if (!it) { - return; + return false; } - for (size_t i = 0; - i < static_cast(OptionalDeviceExtensionVK::kLast); i++) { - it(static_cast(i)); + for (size_t i = 0; i < static_cast(T::kLast); i++) { + if (!it(static_cast(i))) { + return false; + } } + return true; } static std::optional> GetSupportedDeviceExtensions( @@ -200,36 +232,49 @@ CapabilitiesVK::GetEnabledDeviceExtensions( std::vector enabled; - if (exts->find("VK_KHR_swapchain") == exts->end()) { - VALIDATION_LOG << "Device does not support the swapchain extension."; - return std::nullopt; - } - enabled.push_back("VK_KHR_swapchain"); - - // Required for non-conformant implementations like MoltenVK. - if (exts->find("VK_KHR_portability_subset") != exts->end()) { - enabled.push_back("VK_KHR_portability_subset"); - } + auto for_each_common_extension = [&](RequiredCommonDeviceExtensionVK ext) { + auto name = GetExtensionName(ext); + if (exts->find(name) == exts->end()) { + VALIDATION_LOG << "Device does not support required extension: " << name; + return false; + } + enabled.push_back(name); + return true; + }; + auto for_each_android_extension = [&](RequiredAndroidDeviceExtensionVK ext) { #ifdef FML_OS_ANDROID - if (exts->find("VK_ANDROID_external_memory_android_hardware_buffer") == - exts->end()) { - VALIDATION_LOG - << "Device does not support " - "VK_ANDROID_external_memory_android_hardware_buffer extension."; - return std::nullopt; - } - enabled.push_back("VK_ANDROID_external_memory_android_hardware_buffer"); - enabled.push_back("VK_EXT_queue_family_foreign"); -#endif + auto name = GetExtensionName(ext); + if (exts->find(name) == exts->end()) { + VALIDATION_LOG << "Device does not support required Android extension: " + << name; + return false; + } + enabled.push_back(name); +#endif // FML_OS_ANDROID + return true; + }; - // Enable all optional extensions if the device supports it. - IterateOptionalDeviceExtensions([&](auto ext) { - auto ext_name = GetDeviceExtensionName(ext); - if (exts->find(ext_name) != exts->end()) { - enabled.push_back(ext_name); + auto for_each_optional_extension = [&](OptionalDeviceExtensionVK ext) { + auto name = GetExtensionName(ext); + if (exts->find(name) != exts->end()) { + enabled.push_back(name); } - }); + return true; + }; + + const auto iterate_extensions = + IterateExtensions( + for_each_common_extension) && + IterateExtensions( + for_each_android_extension) && + IterateExtensions(for_each_optional_extension); + + if (!iterate_extensions) { + VALIDATION_LOG << "Device not suitable since required extensions are not " + "supported."; + return std::nullopt; + } return enabled; } @@ -282,7 +327,14 @@ static bool HasRequiredQueues(const vk::PhysicalDevice& physical_device) { vk::QueueFlagBits::eTransfer)); } -std::optional +template +static bool IsExtensionInList(const std::vector& list, + ExtensionEnum ext) { + const std::string name = GetExtensionName(ext); + return std::find(list.begin(), list.end(), name) != list.end(); +} + +std::optional CapabilitiesVK::GetEnabledDeviceFeatures( const vk::PhysicalDevice& device) const { if (!PhysicalDeviceSupportsRequiredFormats(device)) { @@ -300,20 +352,41 @@ CapabilitiesVK::GetEnabledDeviceFeatures( return std::nullopt; } - if (!GetEnabledDeviceExtensions(device).has_value()) { + const auto enabled_extensions = GetEnabledDeviceExtensions(device); + if (!enabled_extensions.has_value()) { VALIDATION_LOG << "Device doesn't support the required queues."; return std::nullopt; } - const auto device_features = device.getFeatures(); + PhysicalDeviceFeatures supported_chain; + device.getFeatures2(&supported_chain.get()); - vk::PhysicalDeviceFeatures required; + PhysicalDeviceFeatures required_chain; - // We require this for enabling wireframes in the playground. But its not - // necessarily a big deal if we don't have this feature. - required.fillModeNonSolid = device_features.fillModeNonSolid; + // Base features. + { + auto& required = required_chain.get().features; + const auto& supported = supported_chain.get().features; - return required; + // We require this for enabling wireframes in the playground. But its not + // necessarily a big deal if we don't have this feature. + required.fillModeNonSolid = supported.fillModeNonSolid; + } + // VK_KHR_sampler_ycbcr_conversion features. + if (IsExtensionInList( + enabled_extensions.value(), + RequiredAndroidDeviceExtensionVK::kKHRSamplerYcbcrConversion)) { + auto& required = + required_chain + .get(); + const auto& supported = + supported_chain + .get(); + + required.samplerYcbcrConversion = supported.samplerYcbcrConversion; + } + + return required_chain; } bool CapabilitiesVK::HasLayer(const std::string& layer) const { @@ -386,16 +459,33 @@ bool CapabilitiesVK::SetPhysicalDevice(const vk::PhysicalDevice& device) { // Determine the optional device extensions this physical device supports. { + required_common_device_extensions_.clear(); + required_android_device_extensions_.clear(); optional_device_extensions_.clear(); auto exts = GetSupportedDeviceExtensions(device); if (!exts.has_value()) { return false; } - IterateOptionalDeviceExtensions([&](auto ext) { - auto ext_name = GetDeviceExtensionName(ext); + IterateExtensions([&](auto ext) -> bool { + auto ext_name = GetExtensionName(ext); + if (exts->find(ext_name) != exts->end()) { + required_common_device_extensions_.insert(ext); + } + return true; + }); + IterateExtensions([&](auto ext) -> bool { + auto ext_name = GetExtensionName(ext); + if (exts->find(ext_name) != exts->end()) { + required_android_device_extensions_.insert(ext); + } + return true; + }); + IterateExtensions([&](auto ext) -> bool { + auto ext_name = GetExtensionName(ext); if (exts->find(ext_name) != exts->end()) { optional_device_extensions_.insert(ext); } + return true; }); } @@ -478,14 +568,23 @@ CapabilitiesVK::GetPhysicalDeviceProperties() const { return device_properties_; } -bool CapabilitiesVK::HasOptionalDeviceExtension( - OptionalDeviceExtensionVK extension) const { - return optional_device_extensions_.find(extension) != - optional_device_extensions_.end(); -} - PixelFormat CapabilitiesVK::GetDefaultGlyphAtlasFormat() const { return PixelFormat::kR8UNormInt; } +bool CapabilitiesVK::HasExtension(RequiredCommonDeviceExtensionVK ext) const { + return required_common_device_extensions_.find(ext) != + required_common_device_extensions_.end(); +} + +bool CapabilitiesVK::HasExtension(RequiredAndroidDeviceExtensionVK ext) const { + return required_android_device_extensions_.find(ext) != + required_android_device_extensions_.end(); +} + +bool CapabilitiesVK::HasExtension(OptionalDeviceExtensionVK ext) const { + return optional_device_extensions_.find(ext) != + optional_device_extensions_.end(); +} + } // namespace impeller diff --git a/impeller/renderer/backend/vulkan/capabilities_vk.h b/impeller/renderer/backend/vulkan/capabilities_vk.h index d3f5d70fbffee..f06732b8b206e 100644 --- a/impeller/renderer/backend/vulkan/capabilities_vk.h +++ b/impeller/renderer/backend/vulkan/capabilities_vk.h @@ -19,9 +19,91 @@ namespace impeller { class ContextVK; +//------------------------------------------------------------------------------ +/// @brief A device extension available on all platforms. Without the +/// presence of these extensions, context creation will fail. +/// +enum class RequiredCommonDeviceExtensionVK : uint32_t { + //---------------------------------------------------------------------------- + /// For displaying content in the window system. + /// + /// https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_KHR_swapchain.html + /// + kKHRSwapchain, + + kLast, +}; + +//------------------------------------------------------------------------------ +/// @brief A device extension available on all Android platforms. Without +/// the presence of these extensions on Android, context creation +/// will fail. +/// +/// Platform agnostic code can still check if these Android +/// extensions are present. +/// +enum class RequiredAndroidDeviceExtensionVK : uint32_t { + //---------------------------------------------------------------------------- + /// For importing hardware buffers used in external texture composition. + /// + /// https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_ANDROID_external_memory_android_hardware_buffer.html + /// + kANDROIDExternalMemoryAndroidHardwareBuffer, + + //---------------------------------------------------------------------------- + /// Dependency of kANDROIDExternalMemoryAndroidHardwareBuffer. + /// + /// https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_KHR_sampler_ycbcr_conversion.html + /// + kKHRSamplerYcbcrConversion, + + //---------------------------------------------------------------------------- + /// Dependency of kANDROIDExternalMemoryAndroidHardwareBuffer. + /// + /// https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_KHR_external_memory.html + /// + kKHRExternalMemory, + + //---------------------------------------------------------------------------- + /// Dependency of kANDROIDExternalMemoryAndroidHardwareBuffer. + /// + /// https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_EXT_queue_family_foreign.html + /// + kEXTQueueFamilyForeign, + + //---------------------------------------------------------------------------- + /// Dependency of kANDROIDExternalMemoryAndroidHardwareBuffer. + /// + /// https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_KHR_dedicated_allocation.html + /// + kKHRDedicatedAllocation, + + kLast, +}; + +//------------------------------------------------------------------------------ +/// @brief A device extension enabled if available. Subsystems cannot +/// assume availability and must check if these extensions are +/// available. +/// +/// @see `CapabilitiesVK::HasExtension`. +/// enum class OptionalDeviceExtensionVK : uint32_t { - // https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_EXT_pipeline_creation_feedback.html + //---------------------------------------------------------------------------- + /// To instrument and profile PSO creation. + /// + /// https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_EXT_pipeline_creation_feedback.html + /// kEXTPipelineCreationFeedback, + + //---------------------------------------------------------------------------- + /// To enable context creation on MoltenVK. A non-conformant Vulkan + /// implementation. + /// + /// https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_KHR_portability_subset.html + /// + kVKKHRPortabilitySubset, + kLast, }; @@ -39,7 +121,11 @@ class CapabilitiesVK final : public Capabilities, bool AreValidationsEnabled() const; - bool HasOptionalDeviceExtension(OptionalDeviceExtensionVK extension) const; + bool HasExtension(RequiredCommonDeviceExtensionVK ext) const; + + bool HasExtension(RequiredAndroidDeviceExtensionVK ext) const; + + bool HasExtension(OptionalDeviceExtensionVK ext) const; std::optional> GetEnabledLayers() const; @@ -48,7 +134,11 @@ class CapabilitiesVK final : public Capabilities, std::optional> GetEnabledDeviceExtensions( const vk::PhysicalDevice& physical_device) const; - std::optional GetEnabledDeviceFeatures( + using PhysicalDeviceFeatures = + vk::StructureChain; + + std::optional GetEnabledDeviceFeatures( const vk::PhysicalDevice& physical_device) const; [[nodiscard]] bool SetPhysicalDevice( @@ -106,6 +196,9 @@ class CapabilitiesVK final : public Capabilities, private: bool validations_enabled_ = false; std::map> exts_; + std::set required_common_device_extensions_; + std::set + required_android_device_extensions_; std::set optional_device_extensions_; mutable PixelFormat default_color_format_ = PixelFormat::kUnknown; PixelFormat default_stencil_format_ = PixelFormat::kUnknown; diff --git a/impeller/renderer/backend/vulkan/context_vk.cc b/impeller/renderer/backend/vulkan/context_vk.cc index d32ab4e525b33..9b9caa9cd2d24 100644 --- a/impeller/renderer/backend/vulkan/context_vk.cc +++ b/impeller/renderer/backend/vulkan/context_vk.cc @@ -35,6 +35,7 @@ #include "impeller/renderer/backend/vulkan/gpu_tracer_vk.h" #include "impeller/renderer/backend/vulkan/resource_manager_vk.h" #include "impeller/renderer/backend/vulkan/surface_context_vk.h" +#include "impeller/renderer/backend/vulkan/yuv_conversion_library_vk.h" #include "impeller/renderer/capabilities.h" VULKAN_HPP_DEFAULT_DISPATCH_LOADER_DYNAMIC_STORAGE @@ -323,9 +324,9 @@ void ContextVK::Setup(Settings settings) { vk::DeviceCreateInfo device_info; + device_info.setPNext(&enabled_features.value().get()); device_info.setQueueCreateInfos(queue_create_infos); device_info.setPEnabledExtensionNames(enabled_device_extensions_c); - device_info.setPEnabledFeatures(&enabled_features.value()); // Device layers are deprecated and ignored. { @@ -443,6 +444,8 @@ void ContextVK::Setup(Settings settings) { shader_library_ = std::move(shader_library); sampler_library_ = std::move(sampler_library); pipeline_library_ = std::move(pipeline_library); + yuv_conversion_library_ = std::shared_ptr( + new YUVConversionLibraryVK(device_holder_)); queues_ = std::move(queues); device_capabilities_ = std::move(caps); fence_waiter_ = std::move(fence_waiter); @@ -613,4 +616,9 @@ void ContextVK::InitializeCommonlyUsedShadersIfNeeded() const { auto pass = builder.Build(GetDevice()); } +const std::shared_ptr& +ContextVK::GetYUVConversionLibrary() const { + return yuv_conversion_library_; +} + } // namespace impeller diff --git a/impeller/renderer/backend/vulkan/context_vk.h b/impeller/renderer/backend/vulkan/context_vk.h index 820d9a9c1ed6f..4650ad39e53f9 100644 --- a/impeller/renderer/backend/vulkan/context_vk.h +++ b/impeller/renderer/backend/vulkan/context_vk.h @@ -93,6 +93,9 @@ class ContextVK final : public Context, // |Context| const std::shared_ptr& GetCapabilities() const override; + const std::shared_ptr& GetYUVConversionLibrary() + const; + // |Context| void Shutdown() override; @@ -180,6 +183,7 @@ class ContextVK final : public Context, std::shared_ptr shader_library_; std::shared_ptr sampler_library_; std::shared_ptr pipeline_library_; + std::shared_ptr yuv_conversion_library_; QueuesVK queues_; std::shared_ptr device_capabilities_; std::shared_ptr fence_waiter_; diff --git a/impeller/renderer/backend/vulkan/pipeline_vk.cc b/impeller/renderer/backend/vulkan/pipeline_vk.cc index 13c13ab337a38..eb503e4b83911 100644 --- a/impeller/renderer/backend/vulkan/pipeline_vk.cc +++ b/impeller/renderer/backend/vulkan/pipeline_vk.cc @@ -11,6 +11,7 @@ #include "impeller/renderer/backend/vulkan/context_vk.h" #include "impeller/renderer/backend/vulkan/formats_vk.h" #include "impeller/renderer/backend/vulkan/render_pass_builder_vk.h" +#include "impeller/renderer/backend/vulkan/sampler_vk.h" #include "impeller/renderer/backend/vulkan/shader_function_vk.h" #include "impeller/renderer/backend/vulkan/vertex_descriptor_vk.h" @@ -166,7 +167,8 @@ static vk::UniqueRenderPass CreateCompatRenderPassForPipeline( std::unique_ptr PipelineVK::Create( const PipelineDescriptor& desc, const std::shared_ptr& device_holder, - const std::weak_ptr& weak_library) { + const std::weak_ptr& weak_library, + std::shared_ptr immutable_sampler) { TRACE_EVENT0("flutter", "PipelineVK::Create"); auto library = weak_library.lock(); @@ -183,9 +185,8 @@ std::unique_ptr PipelineVK::Create( const auto* caps = pso_cache->GetCapabilities(); - const auto supports_pipeline_creation_feedback = - caps->HasOptionalDeviceExtension( - OptionalDeviceExtensionVK::kEXTPipelineCreationFeedback); + const auto supports_pipeline_creation_feedback = caps->HasExtension( + OptionalDeviceExtensionVK::kEXTPipelineCreationFeedback); if (!supports_pipeline_creation_feedback) { chain.unlink(); } @@ -353,19 +354,36 @@ std::unique_ptr PipelineVK::Create( //---------------------------------------------------------------------------- /// Pipeline Layout a.k.a the descriptor sets and uniforms. /// - std::vector desc_bindings; + std::vector set_bindings; + + vk::Sampler vk_immutable_sampler = + immutable_sampler ? immutable_sampler->GetSampler() : VK_NULL_HANDLE; for (auto layout : desc.GetVertexDescriptor()->GetDescriptorSetLayouts()) { - auto vk_desc_layout = ToVKDescriptorSetLayoutBinding(layout); - desc_bindings.push_back(vk_desc_layout); + vk::DescriptorSetLayoutBinding set_binding; + set_binding.binding = layout.binding; + set_binding.descriptorCount = 1u; + set_binding.descriptorType = ToVKDescriptorType(layout.descriptor_type); + set_binding.stageFlags = ToVkShaderStage(layout.shader_stage); + // TODO(143719): This specifies the immutable sampler for all sampled + // images. This is incorrect. In cases where the shader samples from the + // multiple images, there is currently no way to tell which sampler needs to + // be immutable and which one needs a binding set in the render pass. Expect + // errors if the shader has more than on sampled image. The sampling from + // the one that is expected to be non-immutable will be incorrect. + if (vk_immutable_sampler && + layout.descriptor_type == DescriptorType::kSampledImage) { + set_binding.setImmutableSamplers(vk_immutable_sampler); + } + set_bindings.push_back(set_binding); } - vk::DescriptorSetLayoutCreateInfo descs_layout_info; - descs_layout_info.setBindings(desc_bindings); + vk::DescriptorSetLayoutCreateInfo desc_set_layout_info; + desc_set_layout_info.setBindings(set_bindings); auto [descs_result, descs_layout] = device_holder->GetDevice().createDescriptorSetLayoutUnique( - descs_layout_info); + desc_set_layout_info); if (descs_result != vk::Result::eSuccess) { VALIDATION_LOG << "unable to create uniform descriptors"; return nullptr; @@ -434,7 +452,8 @@ std::unique_ptr PipelineVK::Create( std::move(pipeline), // std::move(render_pass), // std::move(pipeline_layout.value), // - std::move(descs_layout) // + std::move(descs_layout), // + std::move(immutable_sampler) // )); if (!pipeline_vk->IsValid()) { VALIDATION_LOG << "Could not create a valid pipeline."; @@ -449,13 +468,15 @@ PipelineVK::PipelineVK(std::weak_ptr device_holder, vk::UniquePipeline pipeline, vk::UniqueRenderPass render_pass, vk::UniquePipelineLayout layout, - vk::UniqueDescriptorSetLayout descriptor_set_layout) + vk::UniqueDescriptorSetLayout descriptor_set_layout, + std::shared_ptr immutable_sampler) : Pipeline(std::move(library), desc), device_holder_(std::move(device_holder)), pipeline_(std::move(pipeline)), render_pass_(std::move(render_pass)), layout_(std::move(layout)), - descriptor_set_layout_(std::move(descriptor_set_layout)) { + descriptor_set_layout_(std::move(descriptor_set_layout)), + immutable_sampler_(std::move(immutable_sampler)) { is_valid_ = pipeline_ && render_pass_ && layout_ && descriptor_set_layout_; } @@ -484,4 +505,23 @@ const vk::DescriptorSetLayout& PipelineVK::GetDescriptorSetLayout() const { return *descriptor_set_layout_; } +std::shared_ptr PipelineVK::CreateVariantForImmutableSamplers( + const std::shared_ptr& immutable_sampler) const { + if (!immutable_sampler) { + return nullptr; + } + auto cache_key = ImmutableSamplerKeyVK{*immutable_sampler}; + Lock lock(immutable_sampler_variants_mutex_); + auto found = immutable_sampler_variants_.find(cache_key); + if (found != immutable_sampler_variants_.end()) { + return found->second; + } + auto device_holder = device_holder_.lock(); + if (!device_holder) { + return nullptr; + } + return (immutable_sampler_variants_[cache_key] = + Create(desc_, device_holder, library_, immutable_sampler)); +} + } // namespace impeller diff --git a/impeller/renderer/backend/vulkan/pipeline_vk.h b/impeller/renderer/backend/vulkan/pipeline_vk.h index dfe75cfdafcc1..5040582d23328 100644 --- a/impeller/renderer/backend/vulkan/pipeline_vk.h +++ b/impeller/renderer/backend/vulkan/pipeline_vk.h @@ -10,10 +10,13 @@ #include "impeller/base/backend_cast.h" #include "impeller/base/thread.h" +#include "impeller/core/texture.h" #include "impeller/renderer/backend/vulkan/device_holder.h" #include "impeller/renderer/backend/vulkan/formats_vk.h" #include "impeller/renderer/backend/vulkan/pipeline_cache_vk.h" +#include "impeller/renderer/backend/vulkan/sampler_vk.h" #include "impeller/renderer/backend/vulkan/vk.h" +#include "impeller/renderer/backend/vulkan/yuv_conversion_vk.h" #include "impeller/renderer/pipeline.h" namespace impeller { @@ -29,7 +32,8 @@ class PipelineVK final static std::unique_ptr Create( const PipelineDescriptor& desc, const std::shared_ptr& device_holder, - const std::weak_ptr& weak_library); + const std::weak_ptr& weak_library, + std::shared_ptr immutable_sampler = {}); // |Pipeline| ~PipelineVK() override; @@ -40,15 +44,27 @@ class PipelineVK final const vk::DescriptorSetLayout& GetDescriptorSetLayout() const; + std::shared_ptr CreateVariantForImmutableSamplers( + const std::shared_ptr& immutable_sampler) const; + private: friend class PipelineLibraryVK; + using ImmutableSamplerVariants = + std::unordered_map, + ComparableHash, + ComparableEqual>; + std::weak_ptr device_holder_; vk::UniquePipeline pipeline_; vk::UniqueRenderPass render_pass_; vk::UniquePipelineLayout layout_; vk::UniqueDescriptorSetLayout descriptor_set_layout_; - + std::shared_ptr immutable_sampler_; + mutable Mutex immutable_sampler_variants_mutex_; + mutable ImmutableSamplerVariants immutable_sampler_variants_ IPLR_GUARDED_BY( + immutable_sampler_variants_mutex_); bool is_valid_ = false; PipelineVK(std::weak_ptr device_holder, @@ -57,7 +73,8 @@ class PipelineVK final vk::UniquePipeline pipeline, vk::UniqueRenderPass render_pass, vk::UniquePipelineLayout layout, - vk::UniqueDescriptorSetLayout descriptor_set_layout); + vk::UniqueDescriptorSetLayout descriptor_set_layout, + std::shared_ptr immutable_sampler); // |Pipeline| bool IsValid() const override; diff --git a/impeller/renderer/backend/vulkan/render_pass_vk.cc b/impeller/renderer/backend/vulkan/render_pass_vk.cc index 3149687947696..bc2ed6c5115d7 100644 --- a/impeller/renderer/backend/vulkan/render_pass_vk.cc +++ b/impeller/renderer/backend/vulkan/render_pass_vk.cc @@ -298,26 +298,18 @@ SharedHandleVK RenderPassVK::CreateVKFramebuffer( // |RenderPass| void RenderPassVK::SetPipeline( const std::shared_ptr>& pipeline) { - PipelineVK& pipeline_vk = PipelineVK::Cast(*pipeline); + pipeline_ = pipeline; - auto descriptor_result = - command_buffer_->GetEncoder()->AllocateDescriptorSets( - pipeline_vk.GetDescriptorSetLayout(), ContextVK::Cast(*context_)); - if (!descriptor_result.ok()) { + if (!pipeline_) { return; } - pipeline_valid_ = true; - descriptor_set_ = descriptor_result.value(); - pipeline_layout_ = pipeline_vk.GetPipelineLayout(); - command_buffer_vk_.bindPipeline(vk::PipelineBindPoint::eGraphics, - pipeline_vk.GetPipeline()); pipeline_uses_input_attachments_ = - pipeline->GetDescriptor().GetVertexDescriptor()->UsesInputAttacments(); + pipeline_->GetDescriptor().GetVertexDescriptor()->UsesInputAttacments(); if (pipeline_uses_input_attachments_) { if (bound_image_offset_ >= kMaxBindings) { - pipeline_valid_ = false; + pipeline_ = nullptr; return; } vk::DescriptorImageInfo image_info; @@ -433,14 +425,57 @@ bool RenderPassVK::SetVertexBuffer(VertexBuffer buffer) { // |RenderPass| fml::Status RenderPassVK::Draw() { - if (!pipeline_valid_) { + if (!pipeline_) { return fml::Status(fml::StatusCode::kCancelled, "No valid pipeline is bound to the RenderPass."); } - const ContextVK& context_vk = ContextVK::Cast(*context_); + //---------------------------------------------------------------------------- + /// If there are immutable samplers referenced in the render pass, the base + /// pipeline variant is no longer valid and needs to be re-constructed to + /// reference the samplers. + /// + /// This is an instance of JIT creation of PSOs that can cause jank. It is + /// unavoidable because it isn't possible to know all possible combinations of + /// target YUV conversions. Fortunately, this will only ever happen when + /// rendering to external textures. Like Android Hardware Buffers on Android. + /// + /// Even when JIT creation is unavoidable, pipelines will cache their variants + /// when able and all pipeline creation will happen via a base pipeline cache + /// anyway. So the jank can be mostly entirely ameliorated and it should only + /// ever happen when the first unknown YUV conversion is encountered. + /// + /// Jank can be completely eliminated by pre-populating known YUV conversion + /// pipelines. + if (immutable_sampler_) { + std::shared_ptr pipeline_variant = + PipelineVK::Cast(*pipeline_) + .CreateVariantForImmutableSamplers(immutable_sampler_); + if (!pipeline_variant) { + return fml::Status( + fml::StatusCode::kAborted, + "Could not create pipeline variant with immutable sampler."); + } + pipeline_ = std::move(pipeline_variant); + } + + const auto& context_vk = ContextVK::Cast(*context_); + const auto& pipeline_vk = PipelineVK::Cast(*pipeline_); + + auto descriptor_result = + command_buffer_->GetEncoder()->AllocateDescriptorSets( + pipeline_vk.GetDescriptorSetLayout(), context_vk); + if (!descriptor_result.ok()) { + return fml::Status(fml::StatusCode::kAborted, + "Could not allocate descriptor sets."); + } + const auto descriptor_set = descriptor_result.value(); + const auto pipeline_layout = pipeline_vk.GetPipelineLayout(); + command_buffer_vk_.bindPipeline(vk::PipelineBindPoint::eGraphics, + pipeline_vk.GetPipeline()); + for (auto i = 0u; i < descriptor_write_offset_; i++) { - write_workspace_[i].dstSet = descriptor_set_; + write_workspace_[i].dstSet = descriptor_set; } context_vk.GetDevice().updateDescriptorSets(descriptor_write_offset_, @@ -448,10 +483,10 @@ fml::Status RenderPassVK::Draw() { command_buffer_vk_.bindDescriptorSets( vk::PipelineBindPoint::eGraphics, // bind point - pipeline_layout_, // layout + pipeline_layout, // layout 0, // first set 1, // set count - &descriptor_set_, // sets + &descriptor_set, // sets 0, // offset count nullptr // offsets ); @@ -489,8 +524,9 @@ fml::Status RenderPassVK::Draw() { instance_count_ = 1u; base_vertex_ = 0u; vertex_count_ = 0u; - pipeline_valid_ = false; + pipeline_ = nullptr; pipeline_uses_input_attachments_ = false; + immutable_sampler_ = nullptr; return fml::Status(); } @@ -567,6 +603,10 @@ bool RenderPassVK::BindResource(ShaderStage stage, return false; } + if (!immutable_sampler_) { + immutable_sampler_ = texture_vk.GetImmutableSamplerVariant(sampler_vk); + } + vk::DescriptorImageInfo image_info; image_info.imageLayout = vk::ImageLayout::eShaderReadOnlyOptimal; image_info.sampler = sampler_vk.GetSampler(); diff --git a/impeller/renderer/backend/vulkan/render_pass_vk.h b/impeller/renderer/backend/vulkan/render_pass_vk.h index bf05e523d7ced..49a75fb98598a 100644 --- a/impeller/renderer/backend/vulkan/render_pass_vk.h +++ b/impeller/renderer/backend/vulkan/render_pass_vk.h @@ -16,6 +16,7 @@ namespace impeller { class CommandBufferVK; +class SamplerVK; class RenderPassVK final : public RenderPass { public: @@ -47,10 +48,9 @@ class RenderPassVK final : public RenderPass { size_t vertex_count_ = 0u; bool has_index_buffer_ = false; bool has_label_ = false; - bool pipeline_valid_ = false; + std::shared_ptr> pipeline_; bool pipeline_uses_input_attachments_ = false; - vk::DescriptorSet descriptor_set_ = {}; - vk::PipelineLayout pipeline_layout_ = {}; + std::shared_ptr immutable_sampler_; RenderPassVK(const std::shared_ptr& context, const RenderTarget& target, diff --git a/impeller/renderer/backend/vulkan/sampler_library_vk.cc b/impeller/renderer/backend/vulkan/sampler_library_vk.cc index 81fc820c240ec..97685794df799 100644 --- a/impeller/renderer/backend/vulkan/sampler_library_vk.cc +++ b/impeller/renderer/backend/vulkan/sampler_library_vk.cc @@ -28,41 +28,8 @@ const std::unique_ptr& SamplerLibraryVK::GetSampler( if (!device_holder || !device_holder->GetDevice()) { return kNullSampler; } - - const auto mip_map = ToVKSamplerMipmapMode(desc.mip_filter); - - const auto min_filter = ToVKSamplerMinMagFilter(desc.min_filter); - const auto mag_filter = ToVKSamplerMinMagFilter(desc.mag_filter); - - const auto address_mode_u = ToVKSamplerAddressMode(desc.width_address_mode); - const auto address_mode_v = ToVKSamplerAddressMode(desc.height_address_mode); - const auto address_mode_w = ToVKSamplerAddressMode(desc.depth_address_mode); - - const auto sampler_create_info = - vk::SamplerCreateInfo() - .setMagFilter(mag_filter) - .setMinFilter(min_filter) - .setAddressModeU(address_mode_u) - .setAddressModeV(address_mode_v) - .setAddressModeW(address_mode_w) - .setBorderColor(vk::BorderColor::eFloatTransparentBlack) - .setMipmapMode(mip_map); - - auto res = - device_holder->GetDevice().createSamplerUnique(sampler_create_info); - if (res.result != vk::Result::eSuccess) { - FML_LOG(ERROR) << "Failed to create sampler: " << vk::to_string(res.result); - return kNullSampler; - } - - auto sampler = std::make_unique(desc, std::move(res.value)); - - if (!desc.label.empty()) { - ContextVK::SetDebugName(device_holder->GetDevice(), sampler->GetSampler(), - desc.label.c_str()); - } - - return (samplers_[desc] = std::move(sampler)); + return (samplers_[desc] = + std::make_unique(device_holder->GetDevice(), desc)); } } // namespace impeller diff --git a/impeller/renderer/backend/vulkan/sampler_vk.cc b/impeller/renderer/backend/vulkan/sampler_vk.cc index fd6c1ef542a43..175507d1ab9bf 100644 --- a/impeller/renderer/backend/vulkan/sampler_vk.cc +++ b/impeller/renderer/backend/vulkan/sampler_vk.cc @@ -4,11 +4,106 @@ #include "impeller/renderer/backend/vulkan/sampler_vk.h" +#include "impeller/renderer/backend/vulkan/context_vk.h" +#include "impeller/renderer/backend/vulkan/formats_vk.h" +#include "impeller/renderer/backend/vulkan/yuv_conversion_vk.h" + namespace impeller { -SamplerVK::SamplerVK(SamplerDescriptor desc, vk::UniqueSampler sampler) +static vk::UniqueSampler CreateSampler( + const vk::Device& device, + const SamplerDescriptor& desc, + const std::shared_ptr& yuv_conversion) { + const auto mip_map = ToVKSamplerMipmapMode(desc.mip_filter); + + const auto min_filter = ToVKSamplerMinMagFilter(desc.min_filter); + const auto mag_filter = ToVKSamplerMinMagFilter(desc.mag_filter); + + const auto address_mode_u = ToVKSamplerAddressMode(desc.width_address_mode); + const auto address_mode_v = ToVKSamplerAddressMode(desc.height_address_mode); + const auto address_mode_w = ToVKSamplerAddressMode(desc.depth_address_mode); + + vk::StructureChain + sampler_chain; + + auto& sampler_info = sampler_chain.get(); + + sampler_info.magFilter = mag_filter; + sampler_info.minFilter = min_filter; + sampler_info.addressModeU = address_mode_u; + sampler_info.addressModeV = address_mode_v; + sampler_info.addressModeW = address_mode_w; + sampler_info.borderColor = vk::BorderColor::eFloatTransparentBlack; + sampler_info.mipmapMode = mip_map; + + if (yuv_conversion && yuv_conversion->IsValid()) { + sampler_chain.get().conversion = + yuv_conversion->GetConversion(); + + // + // TL;DR: When using YUV conversion, our samplers are somewhat hobbled and + // not all options configurable in Impeller (especially the linear + // filtering which is by far the most used form of filtering) can be + // supported. Switch to safe defaults. + // + // Spec: If sampler Y'CBCR conversion is enabled and the potential format + // features of the sampler Y'CBCR conversion do not support or enable + // separate reconstruction filters, minFilter and magFilter must be equal to + // the sampler Y'CBCR conversion's chromaFilter. + // + // Thing is, we don't enable separate reconstruction filters. By the time we + // are here, we also don't have access to the descriptor used to create this + // conversion. So we don't yet know what the chromaFilter is. But eNearest + // is a safe bet since the `AndroidHardwareBufferTextureSourceVK` defaults + // to that safe value. So just use that. + // + // See the validation VUID-VkSamplerCreateInfo-minFilter-01645 for more. + // + sampler_info.magFilter = vk::Filter::eNearest; + sampler_info.minFilter = vk::Filter::eNearest; + + // Spec: If sampler Y′CBCR conversion is enabled, addressModeU, + // addressModeV, and addressModeW must be + // VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, anisotropyEnable must be VK_FALSE, + // and unnormalizedCoordinates must be VK_FALSE. + // + // See the validation VUID-VkSamplerCreateInfo-addressModeU-01646 for more. + // + sampler_info.addressModeU = vk::SamplerAddressMode::eClampToEdge; + sampler_info.addressModeV = vk::SamplerAddressMode::eClampToEdge; + sampler_info.addressModeW = vk::SamplerAddressMode::eClampToEdge; + sampler_info.anisotropyEnable = false; + sampler_info.unnormalizedCoordinates = false; + } else { + sampler_chain.unlink(); + } + + auto sampler = device.createSamplerUnique(sampler_chain.get()); + if (sampler.result != vk::Result::eSuccess) { + VALIDATION_LOG << "Could not create sampler: " + << vk::to_string(sampler.result); + return {}; + } + + if (!desc.label.empty()) { + ContextVK::SetDebugName(device, sampler.value.get(), desc.label.c_str()); + } + + return std::move(sampler.value); +} + +SamplerVK::SamplerVK(const vk::Device& device, + SamplerDescriptor desc, + std::shared_ptr yuv_conversion) : Sampler(std::move(desc)), - sampler_(MakeSharedVK(std::move(sampler))) {} + device_(device), + sampler_(MakeSharedVK( + CreateSampler(device, desc_, yuv_conversion))), + yuv_conversion_(std::move(yuv_conversion)) { + is_valid_ = sampler_ && !!sampler_->Get(); +} SamplerVK::~SamplerVK() = default; @@ -16,4 +111,16 @@ vk::Sampler SamplerVK::GetSampler() const { return *sampler_; } +std::shared_ptr SamplerVK::CreateVariantForConversion( + std::shared_ptr conversion) const { + if (!conversion || !is_valid_) { + return nullptr; + } + return std::make_shared(device_, desc_, std::move(conversion)); +} + +const std::shared_ptr& SamplerVK::GetYUVConversion() const { + return yuv_conversion_; +} + } // namespace impeller diff --git a/impeller/renderer/backend/vulkan/sampler_vk.h b/impeller/renderer/backend/vulkan/sampler_vk.h index 30b124a03c682..aa572e1d7a8e5 100644 --- a/impeller/renderer/backend/vulkan/sampler_vk.h +++ b/impeller/renderer/backend/vulkan/sampler_vk.h @@ -14,20 +14,30 @@ namespace impeller { class SamplerLibraryVK; +class YUVConversionVK; class SamplerVK final : public Sampler, public BackendCast { public: - SamplerVK(SamplerDescriptor desc, vk::UniqueSampler sampler); + SamplerVK(const vk::Device& device, + SamplerDescriptor desc, + std::shared_ptr yuv_conversion = {}); // |Sampler| ~SamplerVK() override; vk::Sampler GetSampler() const; + std::shared_ptr CreateVariantForConversion( + std::shared_ptr conversion) const; + + const std::shared_ptr& GetYUVConversion() const; + private: friend SamplerLibraryVK; - std::shared_ptr> sampler_; + const vk::Device device_; + SharedHandleVK sampler_; + std::shared_ptr yuv_conversion_; bool is_valid_ = false; SamplerVK(const SamplerVK&) = delete; diff --git a/impeller/renderer/backend/vulkan/texture_source_vk.cc b/impeller/renderer/backend/vulkan/texture_source_vk.cc index 4a39bce151bd8..8fa29cbe8b9a6 100644 --- a/impeller/renderer/backend/vulkan/texture_source_vk.cc +++ b/impeller/renderer/backend/vulkan/texture_source_vk.cc @@ -14,6 +14,10 @@ const TextureDescriptor& TextureSourceVK::GetTextureDescriptor() const { return desc_; } +std::shared_ptr TextureSourceVK::GetYUVConversion() const { + return nullptr; +} + vk::ImageLayout TextureSourceVK::GetLayout() const { ReaderLock lock(layout_mutex_); return layout_; diff --git a/impeller/renderer/backend/vulkan/texture_source_vk.h b/impeller/renderer/backend/vulkan/texture_source_vk.h index 4f76a8067cf4f..bbe5552870ccf 100644 --- a/impeller/renderer/backend/vulkan/texture_source_vk.h +++ b/impeller/renderer/backend/vulkan/texture_source_vk.h @@ -12,57 +12,117 @@ #include "impeller/renderer/backend/vulkan/formats_vk.h" #include "impeller/renderer/backend/vulkan/shared_object_vk.h" #include "impeller/renderer/backend/vulkan/vk.h" +#include "impeller/renderer/backend/vulkan/yuv_conversion_vk.h" #include "vulkan/vulkan_handles.hpp" namespace impeller { -/// Abstract base class that represents a vkImage and an vkImageView. +//------------------------------------------------------------------------------ +/// @brief Abstract base class that represents a vkImage and an +/// vkImageView. +/// +/// This is intended to be used with an impeller::TextureVK. Example +/// implementations represent swapchain images, uploaded textures, +/// Android Hardware Buffer backend textures, etc... /// -/// This is intended to be used with an impeller::TextureVK. Example -/// implementations represent swapchain images or uploaded textures. class TextureSourceVK { public: virtual ~TextureSourceVK(); + //---------------------------------------------------------------------------- + /// @brief Gets the texture descriptor for this image source. + /// + /// @warning Texture descriptors from texture sources whose capabilities + /// are a superset of those that can be expressed with Vulkan + /// (like Android Hardware Buffer) are inferred. Stuff like size, + /// mip-counts, types is reliable. So use these descriptors as + /// advisory. Creating copies of texture sources from these + /// descriptors is usually not possible and depends on the + /// allocator used. + /// + /// @return The texture descriptor. + /// const TextureDescriptor& GetTextureDescriptor() const; + //---------------------------------------------------------------------------- + /// @brief Get the image handle for this texture source. + /// + /// @return The image. + /// virtual vk::Image GetImage() const = 0; - /// @brief Retrieve the image view used for sampling/blitting/compute with - /// this texture source. + //---------------------------------------------------------------------------- + /// @brief Retrieve the image view used for sampling/blitting/compute + /// with this texture source. + /// + /// @return The image view. + /// virtual vk::ImageView GetImageView() const = 0; - /// @brief Retrieve the image view used for render target attachments - /// with this texture source. + //---------------------------------------------------------------------------- + /// @brief Retrieve the image view used for render target attachments + /// with this texture source. + /// + /// ImageViews used as render target attachments cannot have any + /// mip levels. In cases where we want to generate mipmaps with + /// the result of this texture, we need to create multiple image + /// views. + /// + /// @return The render target view. /// - /// ImageViews used as render target attachments cannot have any mip levels. - /// In cases where we want to generate mipmaps with the result of this - /// texture, we need to create multiple image views. virtual vk::ImageView GetRenderTargetView() const = 0; - /// Encodes the layout transition `barrier` to `barrier.cmd_buffer` for the - /// image. + //---------------------------------------------------------------------------- + /// @brief Encodes the layout transition `barrier` to + /// `barrier.cmd_buffer` for the image. + /// + /// The transition is from the layout stored via + /// `SetLayoutWithoutEncoding` to `barrier.new_layout`. + /// + /// @param[in] barrier The barrier. + /// + /// @return If the layout transition was successfully made. /// - /// The transition is from the layout stored via `SetLayoutWithoutEncoding` to - /// `barrier.new_layout`. fml::Status SetLayout(const BarrierVK& barrier) const; - /// Store the layout of the image. + //---------------------------------------------------------------------------- + /// @brief Store the layout of the image. + /// + /// This just is bookkeeping on the CPU, to actually set the + /// layout use `SetLayout`. /// - /// This just is bookkeeping on the CPU, to actually set the layout use - /// `SetLayout`. + /// @param[in] layout The new layout. + /// + /// @return The old layout. /// - /// @param layout The new layout. - /// @return The old layout. vk::ImageLayout SetLayoutWithoutEncoding(vk::ImageLayout layout) const; - /// Get the last layout assigned to the TextureSourceVK. + //---------------------------------------------------------------------------- + /// @brief Get the last layout assigned to the TextureSourceVK. + /// + /// This value is synchronized with the GPU via SetLayout so it + /// may not reflect the actual layout. + /// + /// @return The last known layout of the texture source. /// - /// This value is synchronized with the GPU via SetLayout so it may not - /// reflect the actual layout. vk::ImageLayout GetLayout() const; - /// Whether or not this is a swapchain image. + //---------------------------------------------------------------------------- + /// @brief When sampling from textures whose formats are not known to + /// Vulkan, a custom conversion is necessary to setup custom + /// samplers. This accessor provides this conversion if one is + /// present. Most texture source have none. + /// + /// @return The sampler conversion. + /// + virtual std::shared_ptr GetYUVConversion() const; + + //---------------------------------------------------------------------------- + /// @brief Determines if swapchain image. That is, an image used as the + /// root render target. + /// + /// @return Whether or not this is a swapchain image. + /// virtual bool IsSwapchainImage() const = 0; protected: diff --git a/impeller/renderer/backend/vulkan/texture_vk.cc b/impeller/renderer/backend/vulkan/texture_vk.cc index b105d31b96e85..aa682b1c5335a 100644 --- a/impeller/renderer/backend/vulkan/texture_vk.cc +++ b/impeller/renderer/backend/vulkan/texture_vk.cc @@ -7,6 +7,7 @@ #include "impeller/renderer/backend/vulkan/command_buffer_vk.h" #include "impeller/renderer/backend/vulkan/command_encoder_vk.h" #include "impeller/renderer/backend/vulkan/formats_vk.h" +#include "impeller/renderer/backend/vulkan/sampler_vk.h" namespace impeller { @@ -191,4 +192,26 @@ SharedHandleVK TextureVK::GetRenderPass() const { return render_pass_; } +void TextureVK::SetMipMapGenerated() { + mipmap_generated_ = true; +} + +bool TextureVK::IsSwapchainImage() const { + return source_->IsSwapchainImage(); +} + +std::shared_ptr TextureVK::GetImmutableSamplerVariant( + const SamplerVK& sampler) const { + if (!source_) { + return nullptr; + } + auto conversion = source_->GetYUVConversion(); + if (!conversion) { + // Most textures don't need a sampler conversion and will go down this path. + // Only needed for YUV sampling from external textures. + return nullptr; + } + return sampler.CreateVariantForConversion(std::move(conversion)); +} + } // namespace impeller diff --git a/impeller/renderer/backend/vulkan/texture_vk.h b/impeller/renderer/backend/vulkan/texture_vk.h index 5826e3df78174..520759e8a6ca3 100644 --- a/impeller/renderer/backend/vulkan/texture_vk.h +++ b/impeller/renderer/backend/vulkan/texture_vk.h @@ -10,6 +10,7 @@ #include "impeller/renderer/backend/vulkan/context_vk.h" #include "impeller/renderer/backend/vulkan/device_buffer_vk.h" #include "impeller/renderer/backend/vulkan/formats_vk.h" +#include "impeller/renderer/backend/vulkan/sampler_vk.h" #include "impeller/renderer/backend/vulkan/texture_source_vk.h" #include "impeller/renderer/backend/vulkan/vk.h" @@ -40,9 +41,12 @@ class TextureVK final : public Texture, public BackendCast { // |Texture| ISize GetSize() const override; - void SetMipMapGenerated() { mipmap_generated_ = true; } + void SetMipMapGenerated(); - bool IsSwapchainImage() const { return source_->IsSwapchainImage(); } + bool IsSwapchainImage() const; + + std::shared_ptr GetImmutableSamplerVariant( + const SamplerVK& sampler) const; // These methods should only be used by render_pass_vk.h diff --git a/impeller/renderer/backend/vulkan/yuv_conversion_library_vk.cc b/impeller/renderer/backend/vulkan/yuv_conversion_library_vk.cc new file mode 100644 index 0000000000000..43417815b2554 --- /dev/null +++ b/impeller/renderer/backend/vulkan/yuv_conversion_library_vk.cc @@ -0,0 +1,34 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "impeller/renderer/backend/vulkan/yuv_conversion_library_vk.h" + +#include "impeller/base/validation.h" +#include "impeller/renderer/backend/vulkan/device_holder.h" + +namespace impeller { + +YUVConversionLibraryVK::YUVConversionLibraryVK( + std::weak_ptr device_holder) + : device_holder_(std::move(device_holder)) {} + +YUVConversionLibraryVK::~YUVConversionLibraryVK() = default; + +std::shared_ptr YUVConversionLibraryVK::GetConversion( + const YUVConversionDescriptorVK& desc) { + Lock lock(conversions_mutex_); + auto found = conversions_.find(desc); + if (found != conversions_.end()) { + return found->second; + } + auto device_holder = device_holder_.lock(); + if (!device_holder) { + VALIDATION_LOG << "Context loss during creation of YUV conversion."; + return nullptr; + } + return (conversions_[desc] = std::shared_ptr( + new YUVConversionVK(device_holder->GetDevice(), desc))); +} + +} // namespace impeller diff --git a/impeller/renderer/backend/vulkan/yuv_conversion_library_vk.h b/impeller/renderer/backend/vulkan/yuv_conversion_library_vk.h new file mode 100644 index 0000000000000..27669ec7c703a --- /dev/null +++ b/impeller/renderer/backend/vulkan/yuv_conversion_library_vk.h @@ -0,0 +1,65 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_IMPELLER_RENDERER_BACKEND_VULKAN_YUV_CONVERSION_LIBRARY_VK_H_ +#define FLUTTER_IMPELLER_RENDERER_BACKEND_VULKAN_YUV_CONVERSION_LIBRARY_VK_H_ + +#include "impeller/renderer/backend/vulkan/yuv_conversion_vk.h" + +namespace impeller { + +class DeviceHolder; + +//------------------------------------------------------------------------------ +/// @brief Due the way the Vulkan spec. treats "identically defined" +/// conversions, creating two conversion with identical descriptors, +/// using one with the image and the other with the sampler, is +/// invalid use. +/// +/// A conversion library hashes and caches identical descriptors to +/// de-duplicate conversions. +/// +/// There can only be one conversion library (the constructor is +/// private to force this) and it found in the context. +/// +class YUVConversionLibraryVK { + public: + ~YUVConversionLibraryVK(); + + YUVConversionLibraryVK(const YUVConversionLibraryVK&) = delete; + + YUVConversionLibraryVK& operator=(const YUVConversionLibraryVK&) = delete; + + //---------------------------------------------------------------------------- + /// @brief Get a conversion for the given descriptor. If there is already + /// a conversion created for an equivalent descriptor, a reference + /// to that descriptor is returned instead. + /// + /// @param[in] desc The descriptor. + /// + /// @return The conversion. A previously created conversion if one was + /// present and a new one if not. A newly created conversion is + /// cached for subsequent accesses. + /// + std::shared_ptr GetConversion( + const YUVConversionDescriptorVK& chain); + + private: + friend class ContextVK; + + using ConversionsMap = std::unordered_map, + YUVConversionDescriptorVKHash, + YUVConversionDescriptorVKEqual>; + + std::weak_ptr device_holder_; + Mutex conversions_mutex_; + ConversionsMap conversions_ IPLR_GUARDED_BY(conversions_mutex_); + + explicit YUVConversionLibraryVK(std::weak_ptr device_holder); +}; + +} // namespace impeller + +#endif // FLUTTER_IMPELLER_RENDERER_BACKEND_VULKAN_YUV_CONVERSION_LIBRARY_VK_H_ diff --git a/impeller/renderer/backend/vulkan/yuv_conversion_vk.cc b/impeller/renderer/backend/vulkan/yuv_conversion_vk.cc new file mode 100644 index 0000000000000..7ed74fb78da07 --- /dev/null +++ b/impeller/renderer/backend/vulkan/yuv_conversion_vk.cc @@ -0,0 +1,118 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "impeller/renderer/backend/vulkan/yuv_conversion_vk.h" + +#include "flutter/fml/hash_combine.h" +#include "impeller/base/validation.h" +#include "impeller/renderer/backend/vulkan/device_holder.h" +#include "impeller/renderer/backend/vulkan/sampler_vk.h" + +namespace impeller { + +YUVConversionVK::YUVConversionVK(const vk::Device& device, + const YUVConversionDescriptorVK& chain) + : chain_(chain) { + auto conversion = device.createSamplerYcbcrConversionUnique(chain_.get()); + if (conversion.result != vk::Result::eSuccess) { + VALIDATION_LOG << "Could not create YUV conversion: " + << vk::to_string(conversion.result); + return; + } + conversion_ = std::move(conversion.value); +} + +YUVConversionVK::~YUVConversionVK() = default; + +bool YUVConversionVK::IsValid() const { + return conversion_ && !!conversion_.get(); +} + +vk::SamplerYcbcrConversion YUVConversionVK::GetConversion() const { + return conversion_ ? conversion_.get() : VK_NULL_HANDLE; +} + +const YUVConversionDescriptorVK& YUVConversionVK::GetDescriptor() const { + return chain_; +} + +std::size_t YUVConversionDescriptorVKHash::operator()( + const YUVConversionDescriptorVK& desc) const { + // Hashers in Vulkan HPP hash the pNext member which isn't what we want for + // these to be stable. + const auto& conv = desc.get(); + + std::size_t hash = fml::HashCombine(conv.format, // + conv.ycbcrModel, // + conv.ycbcrRange, // + conv.components.r, // + conv.components.g, // + conv.components.b, // + conv.components.a, // + conv.xChromaOffset, // + conv.yChromaOffset, // + conv.chromaFilter, // + conv.forceExplicitReconstruction // + ); +#if FML_OS_ANDROID + const auto external_format = desc.get(); + fml::HashCombineSeed(hash, external_format.externalFormat); +#endif // FML_OS_ANDROID + + return hash; +}; + +bool YUVConversionDescriptorVKEqual::operator()( + const YUVConversionDescriptorVK& lhs_desc, + const YUVConversionDescriptorVK& rhs_desc) const { + // Default equality checks in Vulkan HPP checks pNext member members by + // pointer which isn't what we want. + { + const auto& lhs = lhs_desc.get(); + const auto& rhs = rhs_desc.get(); + + if (lhs.format != rhs.format || // + lhs.ycbcrModel != rhs.ycbcrModel || // + lhs.ycbcrRange != rhs.ycbcrRange || // + lhs.components.r != rhs.components.r || // + lhs.components.g != rhs.components.g || // + lhs.components.b != rhs.components.b || // + lhs.components.a != rhs.components.a || // + lhs.xChromaOffset != rhs.xChromaOffset || // + lhs.yChromaOffset != rhs.yChromaOffset || // + lhs.chromaFilter != rhs.chromaFilter || // + lhs.forceExplicitReconstruction != rhs.forceExplicitReconstruction // + ) { + return false; + } + } +#if FML_OS_ANDROID + { + const auto lhs = lhs_desc.get(); + const auto rhs = rhs_desc.get(); + return lhs.externalFormat == rhs.externalFormat; + } +#else // FML_OS_ANDROID + return true; +#endif // FML_OS_ANDROID +} + +ImmutableSamplerKeyVK::ImmutableSamplerKeyVK(const SamplerVK& sampler) + : sampler(sampler.GetDescriptor()) { + if (const auto& conversion = sampler.GetYUVConversion()) { + yuv_conversion = conversion->GetDescriptor(); + } +} + +bool ImmutableSamplerKeyVK::IsEqual(const ImmutableSamplerKeyVK& other) const { + return sampler.IsEqual(other.sampler) && + YUVConversionDescriptorVKEqual{}(yuv_conversion, other.yuv_conversion); +} + +std::size_t ImmutableSamplerKeyVK::GetHash() const { + return fml::HashCombine(sampler.GetHash(), + YUVConversionDescriptorVKHash{}(yuv_conversion)); +} + +} // namespace impeller diff --git a/impeller/renderer/backend/vulkan/yuv_conversion_vk.h b/impeller/renderer/backend/vulkan/yuv_conversion_vk.h new file mode 100644 index 0000000000000..88c17ec6b9ad3 --- /dev/null +++ b/impeller/renderer/backend/vulkan/yuv_conversion_vk.h @@ -0,0 +1,109 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_IMPELLER_RENDERER_BACKEND_VULKAN_YUV_CONVERSION_VK_H_ +#define FLUTTER_IMPELLER_RENDERER_BACKEND_VULKAN_YUV_CONVERSION_VK_H_ + +#include + +#include "flutter/fml/build_config.h" +#include "impeller/base/comparable.h" +#include "impeller/base/thread.h" +#include "impeller/core/sampler.h" +#include "impeller/renderer/backend/vulkan/sampler_vk.h" +#include "impeller/renderer/backend/vulkan/shared_object_vk.h" +#include "impeller/renderer/backend/vulkan/vk.h" + +namespace impeller { + +//------------------------------------------------------------------------------ +/// A descriptor used to create a new YUV conversion in a conversion library. +/// +using YUVConversionDescriptorVK = + vk::StructureChain; + +class YUVConversionLibraryVK; + +//------------------------------------------------------------------------------ +/// @brief It is sometimes necessary to deal with formats not native to +/// Vulkan. In such cases, extra information is necessary to access +/// images. A YUV conversion object is needed in such instances. +/// +/// There are usually only a handful of viable conversions in a +/// given context. However, due to the way the Vulkan spec. treats +/// "identically defined" conversions, only a single conversion +/// object is valid for an equivalent `YUVConversionDescriptorVK`. +/// Because of this restriction, it is not possible to just create a +/// conversion from a descriptor (as the underlying handles will be +/// equivalent but different). Instead, a conversion may only be +/// obtained from a conversion library. Libraries handle hashing and +/// caching conversions by descriptor. Caller can find a library on +/// the top-level context. They may not create their own (the +/// constructor is private). +/// +class YUVConversionVK final { + public: + ~YUVConversionVK(); + + YUVConversionVK(const YUVConversionVK&) = delete; + + YUVConversionVK& operator=(const YUVConversionVK&) = delete; + + //---------------------------------------------------------------------------- + /// @return `true` if this conversion is valid for use with images and + /// samplers. + /// + bool IsValid() const; + + //---------------------------------------------------------------------------- + /// @brief Get the descriptor used to create this conversion. + /// + const YUVConversionDescriptorVK& GetDescriptor() const; + + //---------------------------------------------------------------------------- + /// @return The Vulkan handle of the YUV conversion. + /// + vk::SamplerYcbcrConversion GetConversion() const; + + private: + friend class YUVConversionLibraryVK; + + YUVConversionDescriptorVK chain_; + vk::UniqueSamplerYcbcrConversion conversion_; + + YUVConversionVK(const vk::Device& device, + const YUVConversionDescriptorVK& chain); +}; + +struct YUVConversionDescriptorVKHash { + std::size_t operator()(const YUVConversionDescriptorVK& object) const; +}; + +struct YUVConversionDescriptorVKEqual { + bool operator()(const YUVConversionDescriptorVK& lhs, + const YUVConversionDescriptorVK& rhs) const; +}; + +struct ImmutableSamplerKeyVK : public Comparable { + SamplerDescriptor sampler; + YUVConversionDescriptorVK yuv_conversion; + + explicit ImmutableSamplerKeyVK(const SamplerVK& sampler); + + // |Comparable| + std::size_t GetHash() const override; + + // |Comparable| + bool IsEqual(const ImmutableSamplerKeyVK& other) const override; +}; + +} // namespace impeller + +#endif // FLUTTER_IMPELLER_RENDERER_BACKEND_VULKAN_YUV_CONVERSION_VK_H_ diff --git a/shell/platform/android/image_external_texture_vk.cc b/shell/platform/android/image_external_texture_vk.cc index b5cf834aa9314..906f4a5ee0561 100644 --- a/shell/platform/android/image_external_texture_vk.cc +++ b/shell/platform/android/image_external_texture_vk.cc @@ -54,18 +54,9 @@ void ImageExternalTextureVK::ProcessFrame(PaintContext& context, return; } - impeller::TextureDescriptor desc; - desc.storage_mode = impeller::StorageMode::kDevicePrivate; - desc.size = {static_cast(bounds.width()), - static_cast(bounds.height())}; - // TODO(johnmccutchan): Use hb_desc to compute the correct format at runtime. - desc.format = impeller::PixelFormat::kR8G8B8A8UNormInt; - desc.mip_count = 1; - auto texture_source = std::make_shared( - desc, impeller_context_->GetDevice(), latest_hardware_buffer, - hb_desc); + impeller_context_, latest_hardware_buffer, hb_desc); auto texture = std::make_shared(impeller_context_, texture_source);