From 9756fc93987ca638b96659326d1332df158b6e40 Mon Sep 17 00:00:00 2001 From: jonahwilliams Date: Wed, 15 Feb 2023 18:22:22 -0800 Subject: [PATCH 1/4] [Impeller] improve atlas performance by reducing size of subpass --- impeller/entity/contents/atlas_contents.cc | 185 +++++++++++++++++++-- impeller/entity/contents/atlas_contents.h | 31 ++++ impeller/entity/entity_unittests.cc | 63 +++++++ 3 files changed, 268 insertions(+), 11 deletions(-) diff --git a/impeller/entity/contents/atlas_contents.cc b/impeller/entity/contents/atlas_contents.cc index 26268a1c2f2f9..4ca9d738eb24a 100644 --- a/impeller/entity/contents/atlas_contents.cc +++ b/impeller/entity/contents/atlas_contents.cc @@ -3,8 +3,11 @@ // found in the LICENSE file. #include +#include #include +#include "flutter/fml/macros.h" + #include "impeller/entity/contents/atlas_contents.h" #include "impeller/entity/contents/content_context.h" #include "impeller/entity/contents/filters/color_filter_contents.h" @@ -19,6 +22,10 @@ #include "impeller/renderer/sampler_library.h" #include "impeller/renderer/vertex_buffer_builder.h" +#ifdef FML_OS_PHYSICAL_IOS +#include "impeller/entity/contents/framebuffer_blend_contents.h" +#endif + namespace impeller { AtlasContents::AtlasContents() = default; @@ -57,6 +64,85 @@ void AtlasContents::SetCullRect(std::optional cull_rect) { cull_rect_ = cull_rect; } +struct AtlasBlenderKey { + Color color; + Rect rect; + + struct Hash { + std::size_t operator()(const AtlasBlenderKey& key) const { + return fml::HashCombine(key.color.red, key.color.green, key.color.blue, + key.color.alpha, key.rect.size.width, + key.rect.size.height, key.rect.origin.x, + key.rect.origin.y); + } + }; + + struct Equal { + bool operator()(const AtlasBlenderKey& lhs, + const AtlasBlenderKey& rhs) const { + return lhs.rect == rhs.rect && lhs.color == rhs.color; + } + }; +}; + +std::shared_ptr AtlasContents::GenerateSubAtlas() const { + FML_DCHECK(colors_.size() > 0 && blend_mode_ != BlendMode::kSource && + blend_mode_ != BlendMode::kDestination); + + std::unordered_map, + AtlasBlenderKey::Hash, AtlasBlenderKey::Equal> + sub_atlas = {}; + + for (auto i = 0u; i < texture_coords_.size(); i++) { + AtlasBlenderKey key = {.color = colors_[i], .rect = texture_coords_[i]}; + if (sub_atlas.find(key) == sub_atlas.end()) { + sub_atlas[key] = {transforms_[i]}; + } else { + sub_atlas[key].push_back(transforms_[i]); + } + } + + auto result = std::make_shared(); + Scalar x_offset = 0.0; + Scalar y_offset = 0.0; + Scalar x_extent = 0.0; + Scalar y_extent = 0.0; + + for (auto it = sub_atlas.begin(); it != sub_atlas.end(); it++) { + // This size was arbitrarily chosen to keep the textures from getting too + // wide. We could instead use a more generic rect packer but in the majority + // of cases the sample rects will be fairly close in size making this a good + // enough approximation. + if (x_offset >= 1000) { + y_offset = y_extent + 1; + x_offset = 0.0; + } + + auto key = it->first; + auto transforms = it->second; + + auto new_rect = Rect::MakeXYWH(x_offset, y_offset, key.rect.size.width, + key.rect.size.height); + auto sub_transform = Matrix::MakeTranslation(Vector2(x_offset, y_offset)); + + x_offset += (key.rect.size.width + 1.0); + + result->sub_texture_coords.push_back(key.rect); + result->sub_colors.push_back(key.color); + result->sub_transforms.push_back(sub_transform); + + x_extent = std::max(x_extent, x_offset); + y_extent = std::max(y_extent, y_offset + key.rect.size.height); + + for (auto transform : transforms) { + result->result_texture_coords.push_back(new_rect); + result->result_transforms.push_back(transform); + } + } + result->size = ISize(std::ceil(x_extent), std::ceil(y_extent)); + return result; +} + std::optional AtlasContents::GetCoverage(const Entity& entity) const { if (cull_rect_.has_value()) { return cull_rect_.value().TransformBounds(entity.GetTransformation()); @@ -120,17 +206,56 @@ bool AtlasContents::Render(const ContentContext& renderer, return child_contents.Render(renderer, entity, pass); } + auto sub_atlas = GenerateSubAtlas(); + auto sub_coverage = Rect::MakeSize(sub_atlas->size); + auto src_contents = std::make_shared(*this); - src_contents->SetCoverage(coverage); + src_contents->SetSubAtlas(sub_atlas); + src_contents->SetCoverage(sub_coverage); auto dst_contents = std::make_shared(*this); - dst_contents->SetCoverage(coverage); - + dst_contents->SetSubAtlas(sub_atlas); + dst_contents->SetCoverage(sub_coverage); + +#ifdef FML_OS_PHYSICAL_IOS + auto new_texture = renderer.MakeSubpass( + sub_atlas->size, [&](const ContentContext& context, RenderPass& pass) { + Entity entity; + entity.SetContents(dst_contents); + entity.SetBlendMode(BlendMode::kSource); + if (!entity.Render(context, pass)) { + return false; + } + if (blend_mode_ >= Entity::kLastPipelineBlendMode) { + auto contents = std::make_shared(); + contents->SetBlendMode(blend_mode_); + contents->SetChildContents(src_contents); + entity.SetContents(std::move(contents)); + entity.SetBlendMode(BlendMode::kSource); + return entity.Render(context, pass); + } + entity.SetContents(src_contents); + entity.SetBlendMode(blend_mode_); + return entity.Render(context, pass); + }); +#else auto contents = ColorFilterContents::MakeBlend( blend_mode_, {FilterInput::Make(dst_contents), FilterInput::Make(src_contents)}); - contents->SetAlpha(alpha_); - return contents->Render(renderer, entity, pass); + auto snapshot = contents->RenderToSnapshot(renderer, entity); + if (!snapshot.has_value()) { + return false; + } + auto new_texture = snapshot.value().texture; +#endif + + auto child_contents = AtlasTextureContents(*this); + child_contents.SetAlpha(alpha_); + child_contents.SetCoverage(coverage); + child_contents.SetTexture(new_texture); + child_contents.SetUseDestination(true); + child_contents.SetSubAtlas(sub_atlas); + return child_contents.Render(renderer, entity, pass); } // AtlasTextureContents @@ -154,15 +279,38 @@ void AtlasTextureContents::SetCoverage(Rect coverage) { coverage_ = coverage; } +void AtlasTextureContents::SetUseDestination(bool value) { + use_destination_ = value; +} + +void AtlasTextureContents::SetSubAtlas( + const std::shared_ptr& subatlas) { + subatlas_ = subatlas; +} + +void AtlasTextureContents::SetTexture(std::shared_ptr texture) { + texture_ = std::move(texture); +} + bool AtlasTextureContents::Render(const ContentContext& renderer, const Entity& entity, RenderPass& pass) const { using VS = TextureFillVertexShader; using FS = TextureFillFragmentShader; - auto texture = parent_.GetTexture(); - auto texture_coords = parent_.GetTextureCoordinates(); - auto transforms = parent_.GetTransforms(); + auto texture = texture_.value_or(parent_.GetTexture()); + std::vector texture_coords; + std::vector transforms; + if (subatlas_.has_value()) { + auto subatlas = subatlas_.value(); + texture_coords = use_destination_ ? subatlas->result_texture_coords + : subatlas->sub_texture_coords; + transforms = use_destination_ ? subatlas->result_transforms + : subatlas->sub_transforms; + } else { + texture_coords = parent_.GetTextureCoordinates(); + transforms = parent_.GetTransforms(); + } const auto texture_size = texture->GetSize(); VertexBufferBuilder vertex_builder; @@ -237,15 +385,30 @@ void AtlasColorContents::SetCoverage(Rect coverage) { coverage_ = coverage; } +void AtlasColorContents::SetSubAtlas( + const std::shared_ptr& subatlas) { + subatlas_ = subatlas; +} + bool AtlasColorContents::Render(const ContentContext& renderer, const Entity& entity, RenderPass& pass) const { using VS = GeometryColorPipeline::VertexShader; using FS = GeometryColorPipeline::FragmentShader; - auto texture_coords = parent_.GetTextureCoordinates(); - auto transforms = parent_.GetTransforms(); - auto colors = parent_.GetColors(); + std::vector texture_coords; + std::vector transforms; + std::vector colors; + if (subatlas_.has_value()) { + auto subatlas = subatlas_.value(); + texture_coords = subatlas->sub_texture_coords; + colors = subatlas->sub_colors; + transforms = subatlas->sub_transforms; + } else { + texture_coords = parent_.GetTextureCoordinates(); + transforms = parent_.GetTransforms(); + colors = parent_.GetColors(); + } VertexBufferBuilder vertex_builder; vertex_builder.Reserve(texture_coords.size() * 6); diff --git a/impeller/entity/contents/atlas_contents.h b/impeller/entity/contents/atlas_contents.h index 2313f3402f995..8271242371792 100644 --- a/impeller/entity/contents/atlas_contents.h +++ b/impeller/entity/contents/atlas_contents.h @@ -15,6 +15,20 @@ namespace impeller { +struct SubAtlasResult { + // Sub atlas values. + std::vector sub_texture_coords; + std::vector sub_colors; + std::vector sub_transforms; + + // Result atlas values. + std::vector result_texture_coords; + std::vector result_transforms; + + // Size of the sub-atlass. + ISize size; +}; + class AtlasContents final : public Contents { public: explicit AtlasContents(); @@ -47,6 +61,11 @@ class AtlasContents final : public Contents { const std::vector& GetColors() const; + /// @brief Compress a drawAtlas call with blending into a smaller sized atlas. + /// This atlas has no overlapping to ensure + /// blending behaves as if it were done in the fragment shader. + std::shared_ptr GenerateSubAtlas() const; + // |Contents| std::optional GetCoverage(const Entity& entity) const override; @@ -88,10 +107,19 @@ class AtlasTextureContents final : public Contents { void SetCoverage(Rect coverage); + void SetTexture(std::shared_ptr texture); + + void SetUseDestination(bool value); + + void SetSubAtlas(const std::shared_ptr& subatlas); + private: const AtlasContents& parent_; Scalar alpha_ = 1.0; Rect coverage_; + std::optional> texture_; + bool use_destination_ = false; + std::optional> subatlas_ = std::nullopt; FML_DISALLOW_COPY_AND_ASSIGN(AtlasTextureContents); }; @@ -114,10 +142,13 @@ class AtlasColorContents final : public Contents { void SetCoverage(Rect coverage); + void SetSubAtlas(const std::shared_ptr& subatlas); + private: const AtlasContents& parent_; Scalar alpha_ = 1.0; Rect coverage_; + std::optional> subatlas_ = std::nullopt; FML_DISALLOW_COPY_AND_ASSIGN(AtlasColorContents); }; diff --git a/impeller/entity/entity_unittests.cc b/impeller/entity/entity_unittests.cc index ec12c076dc095..02c55beae0c0f 100644 --- a/impeller/entity/entity_unittests.cc +++ b/impeller/entity/entity_unittests.cc @@ -2069,6 +2069,69 @@ TEST_P(EntityTest, SdfText) { ASSERT_TRUE(OpenPlaygroundHere(callback)); } +TEST_P(EntityTest, AtlasContentsSubAtlas) { + auto boston = CreateTextureForFixture("boston.jpg"); + + { + auto contents = std::make_shared(); + contents->SetBlendMode(BlendMode::kSourceOver); + contents->SetTexture(boston); + contents->SetColors({ + Color::Red(), + Color::Red(), + Color::Red(), + }); + contents->SetTextureCoordinates({ + Rect::MakeLTRB(0, 0, 10, 10), + Rect::MakeLTRB(0, 0, 10, 10), + Rect::MakeLTRB(0, 0, 10, 10), + }); + contents->SetTransforms({ + Matrix::MakeTranslation(Vector2(0, 0)), + Matrix::MakeTranslation(Vector2(100, 100)), + Matrix::MakeTranslation(Vector2(200, 200)), + }); + + // Since all colors and sample rects are the same, there should + // only be a single entry in the sub atlas. + auto subatlas = contents->GenerateSubAtlas(); + ASSERT_EQ(subatlas->sub_texture_coords.size(), 1u); + } + + { + auto contents = std::make_shared(); + contents->SetBlendMode(BlendMode::kSourceOver); + contents->SetTexture(boston); + contents->SetColors({ + Color::Red(), + Color::Green(), + Color::Blue(), + }); + contents->SetTextureCoordinates({ + Rect::MakeLTRB(0, 0, 10, 10), + Rect::MakeLTRB(0, 0, 10, 10), + Rect::MakeLTRB(0, 0, 10, 10), + }); + contents->SetTransforms({ + Matrix::MakeTranslation(Vector2(0, 0)), + Matrix::MakeTranslation(Vector2(100, 100)), + Matrix::MakeTranslation(Vector2(200, 200)), + }); + + // Since all colors are different, there are three entires. + auto subatlas = contents->GenerateSubAtlas(); + ASSERT_EQ(subatlas->sub_texture_coords.size(), 3u); + + // The translations are kept but the sample rects point into + // different parts of the sub atlas. + ASSERT_EQ(subatlas->result_texture_coords[0], Rect::MakeXYWH(0, 0, 10, 10)); + ASSERT_EQ(subatlas->result_texture_coords[1], + Rect::MakeXYWH(11, 0, 10, 10)); + ASSERT_EQ(subatlas->result_texture_coords[2], + Rect::MakeXYWH(22, 0, 10, 10)); + } +} + static Vector3 RGBToYUV(Vector3 rgb, YUVColorSpace yuv_color_space) { Vector3 yuv; switch (yuv_color_space) { From c8f86fee730e73d600d9e3cf5590a530c138e1d5 Mon Sep 17 00:00:00 2001 From: jonahwilliams Date: Thu, 16 Feb 2023 16:31:50 -0800 Subject: [PATCH 2/4] remove usage of floating point color in key --- impeller/entity/contents/atlas_contents.cc | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/impeller/entity/contents/atlas_contents.cc b/impeller/entity/contents/atlas_contents.cc index 4ca9d738eb24a..c04c5854f5e69 100644 --- a/impeller/entity/contents/atlas_contents.cc +++ b/impeller/entity/contents/atlas_contents.cc @@ -67,11 +67,19 @@ void AtlasContents::SetCullRect(std::optional cull_rect) { struct AtlasBlenderKey { Color color; Rect rect; + int32_t color_key; + + static int32_t ToColorKey(Color color) { + return (((std::lround(color.alpha * 255) & 0xff) << 24) | + ((std::lround(color.red * 255) & 0xff) << 16) | + ((std::lround(color.green * 255) & 0xff) << 8) | + ((std::lround(color.blue * 255) & 0xff) << 0)) & + 0xFFFFFFFF; + } struct Hash { std::size_t operator()(const AtlasBlenderKey& key) const { - return fml::HashCombine(key.color.red, key.color.green, key.color.blue, - key.color.alpha, key.rect.size.width, + return fml::HashCombine(key.color_key, key.rect.size.width, key.rect.size.height, key.rect.origin.x, key.rect.origin.y); } @@ -80,7 +88,7 @@ struct AtlasBlenderKey { struct Equal { bool operator()(const AtlasBlenderKey& lhs, const AtlasBlenderKey& rhs) const { - return lhs.rect == rhs.rect && lhs.color == rhs.color; + return lhs.rect == rhs.rect && lhs.color_key == rhs.color_key; } }; }; @@ -94,7 +102,10 @@ std::shared_ptr AtlasContents::GenerateSubAtlas() const { sub_atlas = {}; for (auto i = 0u; i < texture_coords_.size(); i++) { - AtlasBlenderKey key = {.color = colors_[i], .rect = texture_coords_[i]}; + AtlasBlenderKey key = { + .color = colors_[i], + .rect = texture_coords_[i], + .color_key = AtlasBlenderKey::ToColorKey(colors_[i])}; if (sub_atlas.find(key) == sub_atlas.end()) { sub_atlas[key] = {transforms_[i]}; } else { From 3b66c1f1d4fe5a8978ead417a522ac686f6173a9 Mon Sep 17 00:00:00 2001 From: jonahwilliams Date: Fri, 17 Feb 2023 12:01:26 -0800 Subject: [PATCH 3/4] ++ --- impeller/entity/contents/atlas_contents.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/impeller/entity/contents/atlas_contents.cc b/impeller/entity/contents/atlas_contents.cc index c04c5854f5e69..9f0f33631c31b 100644 --- a/impeller/entity/contents/atlas_contents.cc +++ b/impeller/entity/contents/atlas_contents.cc @@ -136,14 +136,14 @@ std::shared_ptr AtlasContents::GenerateSubAtlas() const { key.rect.size.height); auto sub_transform = Matrix::MakeTranslation(Vector2(x_offset, y_offset)); - x_offset += (key.rect.size.width + 1.0); + x_offset += std::ceil(key.rect.size.width) + 1.0; result->sub_texture_coords.push_back(key.rect); result->sub_colors.push_back(key.color); result->sub_transforms.push_back(sub_transform); x_extent = std::max(x_extent, x_offset); - y_extent = std::max(y_extent, y_offset + key.rect.size.height); + y_extent = std::max(y_extent, std::ceil(y_offset + key.rect.size.height)); for (auto transform : transforms) { result->result_texture_coords.push_back(new_rect); From e35cc429e63fbd2d0ba5bf5f01837e0921cb152d Mon Sep 17 00:00:00 2001 From: jonahwilliams Date: Wed, 1 Mar 2023 11:08:59 -0800 Subject: [PATCH 4/4] add helper to color.h --- impeller/display_list/display_list_unittests.cc | 8 ++------ impeller/entity/contents/atlas_contents.cc | 17 ++++------------- impeller/geometry/color.h | 9 +++++++++ impeller/geometry/geometry_unittests.cc | 6 ++++++ 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/impeller/display_list/display_list_unittests.cc b/impeller/display_list/display_list_unittests.cc index 7709025004b4a..0ba6f7a38e9fc 100644 --- a/impeller/display_list/display_list_unittests.cc +++ b/impeller/display_list/display_list_unittests.cc @@ -35,12 +35,8 @@ namespace impeller { namespace testing { flutter::DlColor toColor(const float* components) { - auto value = (((std::lround(components[3] * 255) & 0xff) << 24) | - ((std::lround(components[0] * 255) & 0xff) << 16) | - ((std::lround(components[1] * 255) & 0xff) << 8) | - ((std::lround(components[2] * 255) & 0xff) << 0)) & - 0xFFFFFFFF; - return flutter::DlColor(value); + return flutter::DlColor(Color::ToIColor( + Color(components[0], components[1], components[2], components[3]))); } using DisplayListTest = DisplayListPlayground; diff --git a/impeller/entity/contents/atlas_contents.cc b/impeller/entity/contents/atlas_contents.cc index bda93cdb22aab..99ba234362059 100644 --- a/impeller/entity/contents/atlas_contents.cc +++ b/impeller/entity/contents/atlas_contents.cc @@ -69,15 +69,7 @@ void AtlasContents::SetCullRect(std::optional cull_rect) { struct AtlasBlenderKey { Color color; Rect rect; - int32_t color_key; - - static int32_t ToColorKey(Color color) { - return (((std::lround(color.alpha * 255) & 0xff) << 24) | - ((std::lround(color.red * 255) & 0xff) << 16) | - ((std::lround(color.green * 255) & 0xff) << 8) | - ((std::lround(color.blue * 255) & 0xff) << 0)) & - 0xFFFFFFFF; - } + uint32_t color_key; struct Hash { std::size_t operator()(const AtlasBlenderKey& key) const { @@ -104,10 +96,9 @@ std::shared_ptr AtlasContents::GenerateSubAtlas() const { sub_atlas = {}; for (auto i = 0u; i < texture_coords_.size(); i++) { - AtlasBlenderKey key = { - .color = colors_[i], - .rect = texture_coords_[i], - .color_key = AtlasBlenderKey::ToColorKey(colors_[i])}; + AtlasBlenderKey key = {.color = colors_[i], + .rect = texture_coords_[i], + .color_key = Color::ToIColor(colors_[i])}; if (sub_atlas.find(key) == sub_atlas.end()) { sub_atlas[key] = {transforms_[i]}; } else { diff --git a/impeller/geometry/color.h b/impeller/geometry/color.h index 6966a3ccc0f6d..90a9c49f745c7 100644 --- a/impeller/geometry/color.h +++ b/impeller/geometry/color.h @@ -95,6 +95,15 @@ struct Color { static_cast(b) / 255, static_cast(a) / 255); } + /// @brief Convert this color to a 32-bit representation. + static constexpr uint32_t ToIColor(Color color) { + return (((std::lround(color.alpha * 255) & 0xff) << 24) | + ((std::lround(color.red * 255) & 0xff) << 16) | + ((std::lround(color.green * 255) & 0xff) << 8) | + ((std::lround(color.blue * 255) & 0xff) << 0)) & + 0xFFFFFFFF; + } + constexpr bool operator==(const Color& c) const { return ScalarNearlyEqual(red, c.red) && ScalarNearlyEqual(green, c.green) && ScalarNearlyEqual(blue, c.blue) && ScalarNearlyEqual(alpha, c.alpha); diff --git a/impeller/geometry/geometry_unittests.cc b/impeller/geometry/geometry_unittests.cc index 1e3db566866e7..ca3e02e7eafa8 100644 --- a/impeller/geometry/geometry_unittests.cc +++ b/impeller/geometry/geometry_unittests.cc @@ -1844,6 +1844,12 @@ TEST(GeometryTest, ColorPrinting) { } } +TEST(GeometryTest, ToIColor) { + ASSERT_EQ(Color::ToIColor(Color(0, 0, 0, 0)), 0u); + ASSERT_EQ(Color::ToIColor(Color(1.0, 1.0, 1.0, 1.0)), 0xFFFFFFFF); + ASSERT_EQ(Color::ToIColor(Color(0.5, 0.5, 1.0, 1.0)), 0xFF8080FF); +} + TEST(GeometryTest, Gradient) { { // Simple 2 color gradient produces color buffer containing exactly those