From 2e2cd70ee2df8ae748580f87344e822a1bf40053 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Mon, 22 May 2023 20:43:22 -0700 Subject: [PATCH 01/27] Compiler --- impeller/compiler/impellerc_main.cc | 60 +++++++++++++++++++++++++++++ impeller/compiler/switches.cc | 15 +++++--- impeller/compiler/types.cc | 9 +++++ impeller/compiler/types.h | 13 +++++++ 4 files changed, 92 insertions(+), 5 deletions(-) diff --git a/impeller/compiler/impellerc_main.cc b/impeller/compiler/impellerc_main.cc index 9c7b58b6d760b..3826001ce9498 100644 --- a/impeller/compiler/impellerc_main.cc +++ b/impeller/compiler/impellerc_main.cc @@ -47,6 +47,66 @@ bool Main(const fml::CommandLine& command_line) { return false; } + if (!switches.iplr_bundle.empty()) { + auto json = nlohmann::json::parse(switches.iplr_bundle); + if (!json.is_object()) { + std::cerr << "The IPLR bundle spec must be a JSON object." << std::endl; + return false; + } + + IPLRBundleEntries bundle_entries_result; + for (auto& [bundle_name, bundle_value] : json.items()) { + if (!bundle_value.is_object()) { + std::cerr << "Invalid IPLR bundle entry \"" << bundle_name + << "\": Entry is not a JSON object." << std::endl; + return false; + } + + IPLRBundleEntry bundle_entry_result; + for (auto& [stage_name, stage_value] : bundle_value.items()) { + if (bundle_entry_result.find(stage_name) == bundle_entry_result.end()) { + std::cerr << "Duplicate IPLR bundle stage entry \"" << stage_name + << "\" in bundle \"" << bundle_name << "\"." << std::endl; + return false; + } + if (!bundle_value.is_object()) { + std::cerr << "Invalid IPLR bundle stage entry \"" << stage_name + << "\" in bundle \"" << bundle_name + << "\": Entry is not a JSON object." << std::endl; + return false; + } + + IPLRStageEntry stage_result; + + stage_result.language = stage_value.contains("language") + ? ToSourceLanguage(stage_value["language"]) + : SourceLanguage::kGLSL; + if (stage_result.language == SourceLanguage::kUnknown) { + std::cerr << "Invalid IPLR bundle stage entry \"" << stage_name + << "\" in bundle \"" << bundle_name + << "\": Unknown language type \"" << stage_value["language"] + << "\"." << std::endl; + return false; + } + + if (!stage_value.contains("name")) { + std::cerr << "Duplicate IPLR bundle stage entry \"" << stage_name + << "\" in bundle \"" << bundle_name << "\": Entry ." + << std::endl; + return false; + } + stage_result.source_file_name = stage_value["name"]; + + stage_result.entry_point = stage_value.contains("entry_point") + ? stage_value["entry_point"] + : "main"; + + bundle_entry_result[stage_name] = stage_result; + } + bundle_entries_result[bundle_name] = bundle_entry_result; + } + } + std::shared_ptr source_file_mapping = fml::FileMapping::CreateReadOnly(switches.source_file_name); if (!source_file_mapping) { diff --git a/impeller/compiler/switches.cc b/impeller/compiler/switches.cc index 0e590a5c5c89a..71299e5e89157 100644 --- a/impeller/compiler/switches.cc +++ b/impeller/compiler/switches.cc @@ -146,11 +146,8 @@ Switches::Switches(const fml::CommandLine& command_line) command_line.GetOptionValueWithDefault("source-language", "glsl"); std::transform(language.begin(), language.end(), language.begin(), [](char x) { return std::tolower(x); }); - if (language == "glsl") { - source_language = SourceLanguage::kGLSL; - } else if (language == "hlsl") { - source_language = SourceLanguage::kHLSL; - } + + source_language = ToSourceLanguage(language); if (!working_directory || !working_directory->is_valid()) { return; @@ -228,6 +225,14 @@ bool Switches::AreValid(std::ostream& explain) const { explain << "Spirv file name was empty." << std::endl; valid = false; } + + if (iplr && !iplr_bundle.empty()) { + explain + << "--iplr and --iplr-bundle flag cannot be specified at the same time" + << std::endl; + valid = false; + } + return valid; } diff --git a/impeller/compiler/types.cc b/impeller/compiler/types.cc index 77b40e9b6652e..f492471f40d27 100644 --- a/impeller/compiler/types.cc +++ b/impeller/compiler/types.cc @@ -42,6 +42,15 @@ SourceType SourceTypeFromFileName(const std::string& file_name) { return SourceType::kUnknown; } +SourceLanguage ToSourceLanguage(const std::string& source_language) { + if (source_language == "glsl") { + return SourceLanguage::kGLSL; + } else if (source_language == "hlsl") { + return SourceLanguage::kHLSL; + } + return SourceLanguage::kUnknown; +} + std::string TargetPlatformToString(TargetPlatform platform) { switch (platform) { case TargetPlatform::kUnknown: diff --git a/impeller/compiler/types.h b/impeller/compiler/types.h index 0af2a5c403352..6c1a8ecba9377 100644 --- a/impeller/compiler/types.h +++ b/impeller/compiler/types.h @@ -6,6 +6,7 @@ #include #include +#include #include #include "flutter/fml/macros.h" @@ -42,6 +43,16 @@ enum class SourceLanguage { kHLSL, }; +struct IPLRStageEntry { + std::string source_file_name; + SourceLanguage language; + std::string entry_point; +}; + +using IPLRBundleEntry = std::map; + +using IPLRBundleEntries = std::map; + bool TargetPlatformIsMetal(TargetPlatform platform); bool TargetPlatformIsOpenGL(TargetPlatform platform); @@ -54,6 +65,8 @@ std::string SourceTypeToString(SourceType type); std::string TargetPlatformToString(TargetPlatform platform); +SourceLanguage ToSourceLanguage(const std::string& source_language); + std::string SourceLanguageToString(SourceLanguage source_language); std::string TargetPlatformSLExtension(TargetPlatform platform); From ea36c732c3992c0e4f5faa3e895b3e1bdbaa65fb Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Thu, 29 Jun 2023 15:47:59 -0700 Subject: [PATCH 02/27] Naming --- impeller/compiler/impellerc_main.cc | 43 ++++++++++++++++------------- impeller/compiler/types.h | 5 ++-- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/impeller/compiler/impellerc_main.cc b/impeller/compiler/impellerc_main.cc index 3826001ce9498..18cccbdcd3937 100644 --- a/impeller/compiler/impellerc_main.cc +++ b/impeller/compiler/impellerc_main.cc @@ -50,58 +50,63 @@ bool Main(const fml::CommandLine& command_line) { if (!switches.iplr_bundle.empty()) { auto json = nlohmann::json::parse(switches.iplr_bundle); if (!json.is_object()) { - std::cerr << "The IPLR bundle spec must be a JSON object." << std::endl; + std::cerr << "The shader bundle must be a JSON object." << std::endl; return false; } + /// Bundle parsing. + IPLRBundleEntries bundle_entries_result; for (auto& [bundle_name, bundle_value] : json.items()) { if (!bundle_value.is_object()) { - std::cerr << "Invalid IPLR bundle entry \"" << bundle_name + std::cerr << "Invalid bundle entry \"" << bundle_name << "\": Entry is not a JSON object." << std::endl; return false; } + /// Shader parsing. + IPLRBundleEntry bundle_entry_result; - for (auto& [stage_name, stage_value] : bundle_value.items()) { - if (bundle_entry_result.find(stage_name) == bundle_entry_result.end()) { - std::cerr << "Duplicate IPLR bundle stage entry \"" << stage_name - << "\" in bundle \"" << bundle_name << "\"." << std::endl; + for (auto& [shader_name, shader_value] : bundle_value.items()) { + if (bundle_entry_result.find(shader_name) == + bundle_entry_result.end()) { + std::cerr << "Duplicate shader \"" << shader_name << "\" in bundle \"" + << bundle_name << "\"." << std::endl; return false; } if (!bundle_value.is_object()) { - std::cerr << "Invalid IPLR bundle stage entry \"" << stage_name + std::cerr << "Invalid shader entry \"" << shader_name << "\" in bundle \"" << bundle_name << "\": Entry is not a JSON object." << std::endl; return false; } - IPLRStageEntry stage_result; + IPLRShaderEntry stage_result; - stage_result.language = stage_value.contains("language") - ? ToSourceLanguage(stage_value["language"]) + stage_result.language = shader_value.contains("language") + ? ToSourceLanguage(shader_value["language"]) : SourceLanguage::kGLSL; if (stage_result.language == SourceLanguage::kUnknown) { - std::cerr << "Invalid IPLR bundle stage entry \"" << stage_name + std::cerr << "Invalid shader entry \"" << shader_name << "\" in bundle \"" << bundle_name - << "\": Unknown language type \"" << stage_value["language"] - << "\"." << std::endl; + << "\": Unknown language type \"" + << shader_value["language"] << "\"." << std::endl; return false; } - if (!stage_value.contains("name")) { - std::cerr << "Duplicate IPLR bundle stage entry \"" << stage_name + if (!shader_value.contains("name")) { + std::cerr << "Duplicate IPLR bundle stage entry \"" << shader_name << "\" in bundle \"" << bundle_name << "\": Entry ." << std::endl; return false; } - stage_result.source_file_name = stage_value["name"]; + stage_result.source_file_name = shader_value["name"]; - stage_result.entry_point = stage_value.contains("entry_point") - ? stage_value["entry_point"] + stage_result.entry_point = shader_value.contains("entry_point") + ? shader_value["entry_point"] : "main"; - bundle_entry_result[stage_name] = stage_result; + bundle_entry_result[shader_name] = stage_result; } bundle_entries_result[bundle_name] = bundle_entry_result; } diff --git a/impeller/compiler/types.h b/impeller/compiler/types.h index 6c1a8ecba9377..7cf3b0adf1640 100644 --- a/impeller/compiler/types.h +++ b/impeller/compiler/types.h @@ -43,13 +43,14 @@ enum class SourceLanguage { kHLSL, }; -struct IPLRStageEntry { +struct IPLRShaderEntry { std::string source_file_name; + SourceType type; SourceLanguage language; std::string entry_point; }; -using IPLRBundleEntry = std::map; +using IPLRBundleEntry = std::map; using IPLRBundleEntries = std::map; From e3f8cbff9cf65f2f16558138cce80692abc93d53 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Mon, 4 Dec 2023 23:51:30 -0800 Subject: [PATCH 03/27] [Flutter GPU] Add shader bundle format. --- impeller/compiler/BUILD.gn | 1 + impeller/runtime_stage/BUILD.gn | 16 +++++- impeller/runtime_stage/runtime_stage.cc | 9 +++- impeller/runtime_stage/runtime_stage.fbs | 50 +---------------- impeller/runtime_stage/runtime_stage.h | 5 ++ .../runtime_stage/runtime_stage_types.fbs | 53 +++++++++++++++++++ impeller/shader_bundle/BUILD.gn | 32 +++++++++++ impeller/shader_bundle/shader.cc | 21 ++++++++ impeller/shader_bundle/shader.h | 36 +++++++++++++ impeller/shader_bundle/shader_bundle.cc | 40 ++++++++++++++ impeller/shader_bundle/shader_bundle.fbs | 19 +++++++ impeller/shader_bundle/shader_bundle.h | 35 ++++++++++++ 12 files changed, 266 insertions(+), 51 deletions(-) create mode 100644 impeller/runtime_stage/runtime_stage_types.fbs create mode 100644 impeller/shader_bundle/BUILD.gn create mode 100644 impeller/shader_bundle/shader.cc create mode 100644 impeller/shader_bundle/shader.h create mode 100644 impeller/shader_bundle/shader_bundle.cc create mode 100644 impeller/shader_bundle/shader_bundle.fbs create mode 100644 impeller/shader_bundle/shader_bundle.h diff --git a/impeller/compiler/BUILD.gn b/impeller/compiler/BUILD.gn index b4ff92ee3cb15..7f2c4fd171c84 100644 --- a/impeller/compiler/BUILD.gn +++ b/impeller/compiler/BUILD.gn @@ -65,6 +65,7 @@ impeller_component("compiler_lib") { "../base", "../geometry", "../runtime_stage", + "../shader_bundle", "//flutter/fml", # All third_party deps must be included by the global license script. diff --git a/impeller/runtime_stage/BUILD.gn b/impeller/runtime_stage/BUILD.gn index e8f287132c627..8cc4f3b0a2235 100644 --- a/impeller/runtime_stage/BUILD.gn +++ b/impeller/runtime_stage/BUILD.gn @@ -7,13 +7,25 @@ import("../tools/impeller.gni") config("runtime_stage_config") { configs = [ "//flutter/impeller:impeller_public_config" ] - include_dirs = [ "$root_gen_dir/flutter" ] + include_dirs = [ + "$root_gen_dir/flutter", + "$root_gen_dir/flutter/impeller/runtime_stage", + ] +} + +flatbuffers("runtime_stage_types_flatbuffers") { + flatbuffers = [ "runtime_stage_types.fbs" ] + public_configs = [ ":runtime_stage_config" ] + public_deps = [ "//flutter/third_party/flatbuffers" ] } flatbuffers("runtime_stage_flatbuffers") { flatbuffers = [ "runtime_stage.fbs" ] public_configs = [ ":runtime_stage_config" ] - public_deps = [ "//flutter/third_party/flatbuffers" ] + public_deps = [ + ":runtime_stage_types_flatbuffers", + "//flutter/third_party/flatbuffers", + ] } impeller_component("runtime_stage") { diff --git a/impeller/runtime_stage/runtime_stage.cc b/impeller/runtime_stage/runtime_stage.cc index aed0e01509fcc..cbdee53517ad4 100644 --- a/impeller/runtime_stage/runtime_stage.cc +++ b/impeller/runtime_stage/runtime_stage.cc @@ -63,7 +63,14 @@ RuntimeStage::RuntimeStage(std::shared_ptr payload) if (!fb::RuntimeStageBufferHasIdentifier(payload_->GetMapping())) { return; } - auto runtime_stage = fb::GetRuntimeStage(payload_->GetMapping()); + Setup(fb::GetRuntimeStage(payload_->GetMapping())); +} + +RuntimeStage::RuntimeStage(const fb::RuntimeStage* runtime_stage) { + Setup(runtime_stage); +} + +void RuntimeStage::Setup(const fb::RuntimeStage* runtime_stage) { if (!runtime_stage) { return; } diff --git a/impeller/runtime_stage/runtime_stage.fbs b/impeller/runtime_stage/runtime_stage.fbs index fdb67c71917b5..afb2589cbb18b 100644 --- a/impeller/runtime_stage/runtime_stage.fbs +++ b/impeller/runtime_stage/runtime_stage.fbs @@ -2,55 +2,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -namespace impeller.fb; - -enum Stage:byte { - kVertex, - kFragment, - kCompute, -} - -enum TargetPlatform:byte { - kMetal, - kOpenGLES, - kSkSL, - kVulkan, -} +include "runtime_stage_types.fbs"; -enum UniformDataType:uint32 { - kBoolean, - kSignedByte, - kUnsignedByte, - kSignedShort, - kUnsignedShort, - kSignedInt, - kUnsignedInt, - kSignedInt64, - kUnsignedInt64, - kHalfFloat, - kFloat, - kDouble, - kSampledImage, -} - -table UniformDescription { - name: string; - location: uint64; - type: UniformDataType; - bit_width: uint64; - rows: uint64; - columns: uint64; - array_elements: uint64; -} - -table RuntimeStage { - stage: Stage; - target_platform: TargetPlatform; - entrypoint: string; - uniforms: [UniformDescription]; - shader: [ubyte]; - sksl: [ubyte]; -} +namespace impeller.fb; root_type RuntimeStage; file_identifier "IPLR"; diff --git a/impeller/runtime_stage/runtime_stage.h b/impeller/runtime_stage/runtime_stage.h index 8ee501bbdb650..989b4fab2ac19 100644 --- a/impeller/runtime_stage/runtime_stage.h +++ b/impeller/runtime_stage/runtime_stage.h @@ -11,6 +11,7 @@ #include "flutter/fml/mapping.h" #include "flutter/impeller/core/runtime_types.h" +#include "runtime_stage_types_flatbuffers.h" namespace impeller { @@ -18,6 +19,8 @@ class RuntimeStage { public: explicit RuntimeStage(std::shared_ptr payload); + explicit RuntimeStage(const fb::RuntimeStage* runtime_stage); + ~RuntimeStage(); RuntimeStage(RuntimeStage&&); RuntimeStage& operator=(RuntimeStage&&); @@ -50,6 +53,8 @@ class RuntimeStage { bool is_valid_ = false; bool is_dirty_ = true; + void Setup(const fb::RuntimeStage* runtime_stage); + RuntimeStage(const RuntimeStage&) = delete; RuntimeStage& operator=(const RuntimeStage&) = delete; diff --git a/impeller/runtime_stage/runtime_stage_types.fbs b/impeller/runtime_stage/runtime_stage_types.fbs new file mode 100644 index 0000000000000..c1fb96e3edb88 --- /dev/null +++ b/impeller/runtime_stage/runtime_stage_types.fbs @@ -0,0 +1,53 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +namespace impeller.fb; + +enum Stage:byte { + kVertex, + kFragment, + kCompute, +} + +enum TargetPlatform:byte { + kMetal, + kOpenGLES, + kSkSL, + kVulkan, +} + +enum UniformDataType:uint32 { + kBoolean, + kSignedByte, + kUnsignedByte, + kSignedShort, + kUnsignedShort, + kSignedInt, + kUnsignedInt, + kSignedInt64, + kUnsignedInt64, + kHalfFloat, + kFloat, + kDouble, + kSampledImage, +} + +table UniformDescription { + name: string; + location: uint64; + type: UniformDataType; + bit_width: uint64; + rows: uint64; + columns: uint64; + array_elements: uint64; +} + +table RuntimeStage { + stage: Stage; + target_platform: TargetPlatform; + entrypoint: string; + uniforms: [UniformDescription]; + shader: [ubyte]; + sksl: [ubyte]; +} diff --git a/impeller/shader_bundle/BUILD.gn b/impeller/shader_bundle/BUILD.gn new file mode 100644 index 0000000000000..556a56e480d2a --- /dev/null +++ b/impeller/shader_bundle/BUILD.gn @@ -0,0 +1,32 @@ +# Copyright 2013 The Flutter Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//flutter/third_party/flatbuffers/flatbuffers.gni") +import("../tools/impeller.gni") + +flatbuffers("shader_bundle_flatbuffers") { + flatbuffers = [ "shader_bundle.fbs" ] + public_configs = [ "//flutter/impeller/runtime_stage:runtime_stage_config" ] + public_deps = [ + "//flutter/impeller/runtime_stage:runtime_stage_flatbuffers", + "//flutter/impeller/runtime_stage:runtime_stage_types_flatbuffers", + "//flutter/third_party/flatbuffers", + ] +} + +impeller_component("shader_bundle") { + sources = [ + "shader.cc", + "shader.h", + "shader_bundle.cc", + "shader_bundle.h", + ] + public_deps = [ + ":shader_bundle_flatbuffers", + "../base", + "../core", + "//flutter/fml", + "//flutter/impeller/runtime_stage:runtime_stage", + ] +} diff --git a/impeller/shader_bundle/shader.cc b/impeller/shader_bundle/shader.cc new file mode 100644 index 0000000000000..6194833c59194 --- /dev/null +++ b/impeller/shader_bundle/shader.cc @@ -0,0 +1,21 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "impeller/shader_bundle/shader.h" + +namespace impeller { + +Shader::Shader() = default; + +Shader::Shader(const fb::Shader* shader) + : runtime_stage_( + std::make_shared(RuntimeStage(shader->shader()))) { + is_valid_ = runtime_stage_->IsValid(); +} + +bool Shader::IsValid() const { + return is_valid_; +} + +} // namespace impeller diff --git a/impeller/shader_bundle/shader.h b/impeller/shader_bundle/shader.h new file mode 100644 index 0000000000000..ad639a8360c7a --- /dev/null +++ b/impeller/shader_bundle/shader.h @@ -0,0 +1,36 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#pragma once + +#include +#include + +#include "flutter/fml/mapping.h" +#include "impeller/runtime_stage/runtime_stage.h" +#include "impeller/shader_bundle/shader_bundle_flatbuffers.h" + +namespace impeller { + +class ShaderBundle; + +class Shader { + public: + // Note: Default constructor and copy operations required for map usage in + // ShaderBundle. + Shader(); + + bool IsValid() const; + + private: + bool is_valid_ = false; + + std::shared_ptr runtime_stage_; + + explicit Shader(const fb::Shader* shader); + + friend ShaderBundle; +}; + +} // namespace impeller diff --git a/impeller/shader_bundle/shader_bundle.cc b/impeller/shader_bundle/shader_bundle.cc new file mode 100644 index 0000000000000..1233898fdb737 --- /dev/null +++ b/impeller/shader_bundle/shader_bundle.cc @@ -0,0 +1,40 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "impeller/shader_bundle/shader_bundle.h" + +#include "impeller/shader_bundle/shader_bundle_flatbuffers.h" + +namespace impeller { + +ShaderBundle::ShaderBundle(std::shared_ptr payload) + : payload_(std::move(payload)) { + if (payload_ == nullptr || !payload_->GetMapping()) { + return; + } + if (!fb::ShaderBundleBufferHasIdentifier(payload_->GetMapping())) { + return; + } + auto shader_bundle = fb::GetShaderBundle(payload_->GetMapping()); + if (!shader_bundle) { + return; + } + + auto* shaders = shader_bundle->shaders(); + for (size_t i = 0; i < shaders->size(); i++) { + const fb::Shader* shader = shaders->Get(i); + shaders_[shader->name()->str()] = Shader(shader); + if (!shaders_[shader->name()->str()].IsValid()) { + return; + } + } + + is_valid_ = true; +} + +bool ShaderBundle::IsValid() const { + return is_valid_; +} + +} // namespace impeller diff --git a/impeller/shader_bundle/shader_bundle.fbs b/impeller/shader_bundle/shader_bundle.fbs new file mode 100644 index 0000000000000..1c0f4ce038967 --- /dev/null +++ b/impeller/shader_bundle/shader_bundle.fbs @@ -0,0 +1,19 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +include "../runtime_stage/runtime_stage_types.fbs"; + +namespace impeller.fb; + +table Shader { + name: string; + shader: RuntimeStage; +} + +table ShaderBundle { + shaders: [Shader]; +} + +root_type ShaderBundle; +file_identifier "IPSB"; diff --git a/impeller/shader_bundle/shader_bundle.h b/impeller/shader_bundle/shader_bundle.h new file mode 100644 index 0000000000000..f48a33bf2a674 --- /dev/null +++ b/impeller/shader_bundle/shader_bundle.h @@ -0,0 +1,35 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#pragma once + +#include +#include + +#include "flutter/fml/mapping.h" +#include "impeller/runtime_stage/runtime_stage.h" +#include "impeller/shader_bundle/shader.h" + +namespace impeller { + +class ShaderBundle { + public: + explicit ShaderBundle(std::shared_ptr payload); + + ~ShaderBundle(); + ShaderBundle(ShaderBundle&&); + + bool IsValid() const; + + private: + bool is_valid_; + + std::shared_ptr payload_; + std::map shaders_; + + ShaderBundle(const ShaderBundle&) = delete; + ShaderBundle& operator=(const ShaderBundle&) = delete; +}; + +} // namespace impeller From f8e470339a2c61f096debc4ac14a7d0045d0ca2c Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Tue, 5 Dec 2023 01:05:01 -0800 Subject: [PATCH 04/27] Split compiler frontend code for readability --- impeller/compiler/impellerc_main.cc | 287 ++++++++++++++++------------ 1 file changed, 160 insertions(+), 127 deletions(-) diff --git a/impeller/compiler/impellerc_main.cc b/impeller/compiler/impellerc_main.cc index 18cccbdcd3937..587da417c7754 100644 --- a/impeller/compiler/impellerc_main.cc +++ b/impeller/compiler/impellerc_main.cc @@ -33,6 +33,162 @@ static bool SetPermissiveAccess(const std::filesystem::path& p) { return true; } +/// Run the shader compiler to geneate SkSL. +/// If there is an error, prints error text and returns `nullptr`. +static std::shared_ptr CompileSkSL( + std::shared_ptr source_file_mapping, + SourceOptions& options, + Reflector::Options& reflector_options) { + SourceOptions sksl_options = options; + sksl_options.target_platform = TargetPlatform::kSkSL; + + Reflector::Options sksl_reflector_options = reflector_options; + sksl_reflector_options.target_platform = TargetPlatform::kSkSL; + + Compiler sksl_compiler = Compiler(std::move(source_file_mapping), + sksl_options, sksl_reflector_options); + if (!sksl_compiler.IsValid()) { + std::cerr << "Compilation to SkSL failed." << std::endl; + std::cerr << sksl_compiler.GetErrorMessages() << std::endl; + return nullptr; + } + return sksl_compiler.GetSLShaderSource(); +} + +/// Outputs artifacts for a single compiler invocation and option configuration. +/// If there is an error, prints error text and returns `false`. +static bool OutputArtifacts(Compiler& compiler, + Switches& switches, + SourceOptions& options, + std::shared_ptr sksl_mapping) { + /// 1. Output the source file. When in IPLR/RuntimeStage mode, this is the + /// IPLR flatbuffer file. + + auto sl_file_name = std::filesystem::absolute( + std::filesystem::current_path() / switches.sl_file_name); + if (switches.iplr) { + auto reflector = compiler.GetReflector(); + if (reflector == nullptr) { + std::cerr << "Could not create reflector." << std::endl; + return false; + } + auto stage_data = reflector->GetRuntimeStageData(); + if (!stage_data) { + std::cerr << "Runtime stage information was nil." << std::endl; + return false; + } + if (sksl_mapping) { + stage_data->SetSkSLData(std::move(sksl_mapping)); + } + auto stage_data_mapping = options.json_format + ? stage_data->CreateJsonMapping() + : stage_data->CreateMapping(); + if (!stage_data_mapping) { + std::cerr << "Runtime stage data could not be created." << std::endl; + return false; + } + if (!fml::WriteAtomically(*switches.working_directory, // + Utf8FromPath(sl_file_name).c_str(), // + *stage_data_mapping // + )) { + std::cerr << "Could not write file to " << switches.sl_file_name + << std::endl; + return false; + } + // Tools that consume the runtime stage data expect the access mode to + // be 0644. + if (!SetPermissiveAccess(sl_file_name)) { + return false; + } + } else { + if (!fml::WriteAtomically(*switches.working_directory, + Utf8FromPath(sl_file_name).c_str(), + *compiler.GetSLShaderSource())) { + std::cerr << "Could not write file to " << switches.sl_file_name + << std::endl; + return false; + } + } + + /// 2. Output shader reflection data. + /// May include a JSON file, a C++ header, and/or a C++ TU. + + if (TargetPlatformNeedsReflection(options.target_platform)) { + if (!switches.reflection_json_name.empty()) { + auto reflection_json_name = std::filesystem::absolute( + std::filesystem::current_path() / switches.reflection_json_name); + if (!fml::WriteAtomically( + *switches.working_directory, + Utf8FromPath(reflection_json_name).c_str(), + *compiler.GetReflector()->GetReflectionJSON())) { + std::cerr << "Could not write reflection json to " + << switches.reflection_json_name << std::endl; + return false; + } + } + + if (!switches.reflection_header_name.empty()) { + auto reflection_header_name = + std::filesystem::absolute(std::filesystem::current_path() / + switches.reflection_header_name.c_str()); + if (!fml::WriteAtomically( + *switches.working_directory, + Utf8FromPath(reflection_header_name).c_str(), + *compiler.GetReflector()->GetReflectionHeader())) { + std::cerr << "Could not write reflection header to " + << switches.reflection_header_name << std::endl; + return false; + } + } + + if (!switches.reflection_cc_name.empty()) { + auto reflection_cc_name = + std::filesystem::absolute(std::filesystem::current_path() / + switches.reflection_cc_name.c_str()); + if (!fml::WriteAtomically(*switches.working_directory, + Utf8FromPath(reflection_cc_name).c_str(), + *compiler.GetReflector()->GetReflectionCC())) { + std::cerr << "Could not write reflection CC to " + << switches.reflection_cc_name << std::endl; + return false; + } + } + } + + /// 3. Output a depfile. + + if (!switches.depfile_path.empty()) { + std::string result_file; + switch (switches.target_platform) { + case TargetPlatform::kMetalDesktop: + case TargetPlatform::kMetalIOS: + case TargetPlatform::kOpenGLES: + case TargetPlatform::kOpenGLDesktop: + case TargetPlatform::kRuntimeStageMetal: + case TargetPlatform::kRuntimeStageGLES: + case TargetPlatform::kRuntimeStageVulkan: + case TargetPlatform::kSkSL: + case TargetPlatform::kVulkan: + result_file = switches.sl_file_name; + break; + case TargetPlatform::kUnknown: + result_file = switches.spirv_file_name; + break; + } + auto depfile_path = std::filesystem::absolute( + std::filesystem::current_path() / switches.depfile_path.c_str()); + if (!fml::WriteAtomically(*switches.working_directory, + Utf8FromPath(depfile_path).c_str(), + *compiler.CreateDepfileContents({result_file}))) { + std::cerr << "Could not write depfile to " << switches.depfile_path + << std::endl; + return false; + } + } + + return true; +} + bool Main(const fml::CommandLine& command_line) { fml::InstallCrashHandler(); if (command_line.HasOption("help")) { @@ -152,20 +308,10 @@ bool Main(const fml::CommandLine& command_line) { std::shared_ptr sksl_mapping; if (switches.iplr && TargetPlatformBundlesSkSL(switches.target_platform) && switches.iplr_bundle.empty()) { - SourceOptions sksl_options = options; - sksl_options.target_platform = TargetPlatform::kSkSL; - - Reflector::Options sksl_reflector_options = reflector_options; - sksl_reflector_options.target_platform = TargetPlatform::kSkSL; - - Compiler sksl_compiler = - Compiler(source_file_mapping, sksl_options, sksl_reflector_options); - if (!sksl_compiler.IsValid()) { - std::cerr << "Compilation to SkSL failed." << std::endl; - std::cerr << sksl_compiler.GetErrorMessages() << std::endl; + sksl_mapping = CompileSkSL(source_file_mapping, options, reflector_options); + if (!sksl_mapping) { return false; } - sksl_mapping = sksl_compiler.GetSLShaderSource(); } Compiler compiler(source_file_mapping, options, reflector_options); @@ -185,121 +331,8 @@ bool Main(const fml::CommandLine& command_line) { return false; } - auto sl_file_name = std::filesystem::absolute( - std::filesystem::current_path() / switches.sl_file_name); - if (switches.iplr) { - auto reflector = compiler.GetReflector(); - if (reflector == nullptr) { - std::cerr << "Could not create reflector." << std::endl; - return false; - } - auto stage_data = reflector->GetRuntimeStageData(); - if (!stage_data) { - std::cerr << "Runtime stage information was nil." << std::endl; - return false; - } - if (sksl_mapping) { - stage_data->SetSkSLData(sksl_mapping); - } - auto stage_data_mapping = options.json_format - ? stage_data->CreateJsonMapping() - : stage_data->CreateMapping(); - if (!stage_data_mapping) { - std::cerr << "Runtime stage data could not be created." << std::endl; - return false; - } - if (!fml::WriteAtomically(*switches.working_directory, // - Utf8FromPath(sl_file_name).c_str(), // - *stage_data_mapping // - )) { - std::cerr << "Could not write file to " << switches.sl_file_name - << std::endl; - return false; - } - // Tools that consume the runtime stage data expect the access mode to - // be 0644. - if (!SetPermissiveAccess(sl_file_name)) { - return false; - } - } else { - if (!fml::WriteAtomically(*switches.working_directory, - Utf8FromPath(sl_file_name).c_str(), - *compiler.GetSLShaderSource())) { - std::cerr << "Could not write file to " << switches.sl_file_name - << std::endl; - return false; - } - } - - if (TargetPlatformNeedsReflection(options.target_platform)) { - if (!switches.reflection_json_name.empty()) { - auto reflection_json_name = std::filesystem::absolute( - std::filesystem::current_path() / switches.reflection_json_name); - if (!fml::WriteAtomically( - *switches.working_directory, - Utf8FromPath(reflection_json_name).c_str(), - *compiler.GetReflector()->GetReflectionJSON())) { - std::cerr << "Could not write reflection json to " - << switches.reflection_json_name << std::endl; - return false; - } - } - - if (!switches.reflection_header_name.empty()) { - auto reflection_header_name = - std::filesystem::absolute(std::filesystem::current_path() / - switches.reflection_header_name.c_str()); - if (!fml::WriteAtomically( - *switches.working_directory, - Utf8FromPath(reflection_header_name).c_str(), - *compiler.GetReflector()->GetReflectionHeader())) { - std::cerr << "Could not write reflection header to " - << switches.reflection_header_name << std::endl; - return false; - } - } - - if (!switches.reflection_cc_name.empty()) { - auto reflection_cc_name = - std::filesystem::absolute(std::filesystem::current_path() / - switches.reflection_cc_name.c_str()); - if (!fml::WriteAtomically(*switches.working_directory, - Utf8FromPath(reflection_cc_name).c_str(), - *compiler.GetReflector()->GetReflectionCC())) { - std::cerr << "Could not write reflection CC to " - << switches.reflection_cc_name << std::endl; - return false; - } - } - } - - if (!switches.depfile_path.empty()) { - std::string result_file; - switch (switches.target_platform) { - case TargetPlatform::kMetalDesktop: - case TargetPlatform::kMetalIOS: - case TargetPlatform::kOpenGLES: - case TargetPlatform::kOpenGLDesktop: - case TargetPlatform::kRuntimeStageMetal: - case TargetPlatform::kRuntimeStageGLES: - case TargetPlatform::kRuntimeStageVulkan: - case TargetPlatform::kSkSL: - case TargetPlatform::kVulkan: - result_file = switches.sl_file_name; - break; - case TargetPlatform::kUnknown: - result_file = switches.spirv_file_name; - break; - } - auto depfile_path = std::filesystem::absolute( - std::filesystem::current_path() / switches.depfile_path.c_str()); - if (!fml::WriteAtomically(*switches.working_directory, - Utf8FromPath(depfile_path).c_str(), - *compiler.CreateDepfileContents({result_file}))) { - std::cerr << "Could not write depfile to " << switches.depfile_path - << std::endl; - return false; - } + if (!OutputArtifacts(compiler, switches, options, sksl_mapping)) { + return false; } return true; From 91cfa250123398518857eecb4a9fb648c8f70afb Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Tue, 5 Dec 2023 01:34:35 -0800 Subject: [PATCH 05/27] Move bundle parser into its own file --- impeller/compiler/BUILD.gn | 2 + impeller/compiler/impellerc_main.cc | 102 +++++++--------------------- impeller/compiler/shader_bundle.cc | 81 ++++++++++++++++++++++ impeller/compiler/shader_bundle.h | 13 ++++ 4 files changed, 121 insertions(+), 77 deletions(-) create mode 100644 impeller/compiler/shader_bundle.cc create mode 100644 impeller/compiler/shader_bundle.h diff --git a/impeller/compiler/BUILD.gn b/impeller/compiler/BUILD.gn index 7f2c4fd171c84..d279984ea73fe 100644 --- a/impeller/compiler/BUILD.gn +++ b/impeller/compiler/BUILD.gn @@ -46,6 +46,8 @@ impeller_component("compiler_lib") { "reflector.h", "runtime_stage_data.cc", "runtime_stage_data.h", + "shader_bundle.cc", + "shader_bundle.h", "source_options.cc", "source_options.h", "spirv_compiler.cc", diff --git a/impeller/compiler/impellerc_main.cc b/impeller/compiler/impellerc_main.cc index 587da417c7754..68616afd929b7 100644 --- a/impeller/compiler/impellerc_main.cc +++ b/impeller/compiler/impellerc_main.cc @@ -10,6 +10,7 @@ #include "flutter/fml/file.h" #include "flutter/fml/mapping.h" #include "impeller/compiler/compiler.h" +#include "impeller/compiler/shader_bundle.h" #include "impeller/compiler/source_options.h" #include "impeller/compiler/switches.h" #include "impeller/compiler/types.h" @@ -59,9 +60,21 @@ static std::shared_ptr CompileSkSL( /// If there is an error, prints error text and returns `false`. static bool OutputArtifacts(Compiler& compiler, Switches& switches, + std::shared_ptr source_file_mapping, SourceOptions& options, - std::shared_ptr sksl_mapping) { - /// 1. Output the source file. When in IPLR/RuntimeStage mode, this is the + Reflector::Options& reflector_options) { + /// 1. Invoke the compiler to generate SkSL if needed. + + std::shared_ptr sksl_mapping; + if (switches.iplr && TargetPlatformBundlesSkSL(switches.target_platform)) { + sksl_mapping = + CompileSkSL(std::move(source_file_mapping), options, reflector_options); + if (!sksl_mapping) { + return false; + } + } + + /// 2. Output the source file. When in IPLR/RuntimeStage mode, this is the /// IPLR flatbuffer file. auto sl_file_name = std::filesystem::absolute( @@ -110,7 +123,7 @@ static bool OutputArtifacts(Compiler& compiler, } } - /// 2. Output shader reflection data. + /// 3. Output shader reflection data. /// May include a JSON file, a C++ header, and/or a C++ TU. if (TargetPlatformNeedsReflection(options.target_platform)) { @@ -155,7 +168,7 @@ static bool OutputArtifacts(Compiler& compiler, } } - /// 3. Output a depfile. + /// 4. Output a depfile. if (!switches.depfile_path.empty()) { std::string result_file; @@ -204,70 +217,14 @@ bool Main(const fml::CommandLine& command_line) { } if (!switches.iplr_bundle.empty()) { - auto json = nlohmann::json::parse(switches.iplr_bundle); - if (!json.is_object()) { - std::cerr << "The shader bundle must be a JSON object." << std::endl; - return false; - } - - /// Bundle parsing. - - IPLRBundleEntries bundle_entries_result; - for (auto& [bundle_name, bundle_value] : json.items()) { - if (!bundle_value.is_object()) { - std::cerr << "Invalid bundle entry \"" << bundle_name - << "\": Entry is not a JSON object." << std::endl; - return false; - } - - /// Shader parsing. - - IPLRBundleEntry bundle_entry_result; - for (auto& [shader_name, shader_value] : bundle_value.items()) { - if (bundle_entry_result.find(shader_name) == - bundle_entry_result.end()) { - std::cerr << "Duplicate shader \"" << shader_name << "\" in bundle \"" - << bundle_name << "\"." << std::endl; - return false; - } - if (!bundle_value.is_object()) { - std::cerr << "Invalid shader entry \"" << shader_name - << "\" in bundle \"" << bundle_name - << "\": Entry is not a JSON object." << std::endl; - return false; - } - - IPLRShaderEntry stage_result; - - stage_result.language = shader_value.contains("language") - ? ToSourceLanguage(shader_value["language"]) - : SourceLanguage::kGLSL; - if (stage_result.language == SourceLanguage::kUnknown) { - std::cerr << "Invalid shader entry \"" << shader_name - << "\" in bundle \"" << bundle_name - << "\": Unknown language type \"" - << shader_value["language"] << "\"." << std::endl; - return false; - } - - if (!shader_value.contains("name")) { - std::cerr << "Duplicate IPLR bundle stage entry \"" << shader_name - << "\" in bundle \"" << bundle_name << "\": Entry ." - << std::endl; - return false; - } - stage_result.source_file_name = shader_value["name"]; - - stage_result.entry_point = shader_value.contains("entry_point") - ? shader_value["entry_point"] - : "main"; - - bundle_entry_result[shader_name] = stage_result; - } - bundle_entries_result[bundle_name] = bundle_entry_result; - } + // Invoke the compiler multiple times to build a shader bundle with the + // given iplr_bundle spec. + return GenerateShaderBundle(switches); } + // Invoke the compiler and generate reflection data for a single shader or + // runtime stage IPLR. + std::shared_ptr source_file_mapping = fml::FileMapping::CreateReadOnly(switches.source_file_name); if (!source_file_mapping) { @@ -304,16 +261,6 @@ bool Main(const fml::CommandLine& command_line) { reflector_options.header_file_name = Utf8FromPath( std::filesystem::path{switches.reflection_header_name}.filename()); - // Generate SkSL if needed. - std::shared_ptr sksl_mapping; - if (switches.iplr && TargetPlatformBundlesSkSL(switches.target_platform) && - switches.iplr_bundle.empty()) { - sksl_mapping = CompileSkSL(source_file_mapping, options, reflector_options); - if (!sksl_mapping) { - return false; - } - } - Compiler compiler(source_file_mapping, options, reflector_options); if (!compiler.IsValid()) { std::cerr << "Compilation failed." << std::endl; @@ -331,7 +278,8 @@ bool Main(const fml::CommandLine& command_line) { return false; } - if (!OutputArtifacts(compiler, switches, options, sksl_mapping)) { + if (!OutputArtifacts(compiler, switches, std::move(source_file_mapping), + options, reflector_options)) { return false; } diff --git a/impeller/compiler/shader_bundle.cc b/impeller/compiler/shader_bundle.cc new file mode 100644 index 0000000000000..780cc092a99a0 --- /dev/null +++ b/impeller/compiler/shader_bundle.cc @@ -0,0 +1,81 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "impeller/compiler/shader_bundle.h" +#include "impeller/compiler/types.h" + +#include "third_party/json/include/nlohmann/json.hpp" + +namespace impeller { +namespace compiler { + +/// Compiles and outputs a +bool GenerateShaderBundle(Switches& switches) { + auto json = nlohmann::json::parse(switches.iplr_bundle); + if (!json.is_object()) { + std::cerr << "The shader bundle must be a JSON object." << std::endl; + return false; + } + + /// Bundle parsing. + + IPLRBundleEntries bundle_entries_result; + for (auto& [bundle_name, bundle_value] : json.items()) { + if (!bundle_value.is_object()) { + std::cerr << "Invalid bundle entry \"" << bundle_name + << "\": Entry is not a JSON object." << std::endl; + return false; + } + + /// Shader parsing. + + IPLRBundleEntry bundle_entry_result; + for (auto& [shader_name, shader_value] : bundle_value.items()) { + if (bundle_entry_result.find(shader_name) == bundle_entry_result.end()) { + std::cerr << "Duplicate shader \"" << shader_name << "\" in bundle \"" + << bundle_name << "\"." << std::endl; + return false; + } + if (!bundle_value.is_object()) { + std::cerr << "Invalid shader entry \"" << shader_name + << "\" in bundle \"" << bundle_name + << "\": Entry is not a JSON object." << std::endl; + return false; + } + + IPLRShaderEntry stage_result; + + stage_result.language = shader_value.contains("language") + ? ToSourceLanguage(shader_value["language"]) + : SourceLanguage::kGLSL; + if (stage_result.language == SourceLanguage::kUnknown) { + std::cerr << "Invalid shader entry \"" << shader_name + << "\" in bundle \"" << bundle_name + << "\": Unknown language type \"" << shader_value["language"] + << "\"." << std::endl; + return false; + } + + if (!shader_value.contains("name")) { + std::cerr << "Duplicate IPLR bundle stage entry \"" << shader_name + << "\" in bundle \"" << bundle_name << "\": Entry ." + << std::endl; + return false; + } + stage_result.source_file_name = shader_value["name"]; + + stage_result.entry_point = shader_value.contains("entry_point") + ? shader_value["entry_point"] + : "main"; + + bundle_entry_result[shader_name] = stage_result; + } + bundle_entries_result[bundle_name] = bundle_entry_result; + } + + return true; +} + +} // namespace compiler +} // namespace impeller diff --git a/impeller/compiler/shader_bundle.h b/impeller/compiler/shader_bundle.h new file mode 100644 index 0000000000000..d364561c2184b --- /dev/null +++ b/impeller/compiler/shader_bundle.h @@ -0,0 +1,13 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "impeller/compiler/switches.h" + +namespace impeller { +namespace compiler { + +bool GenerateShaderBundle(Switches& switches); + +} // namespace compiler +} // namespace impeller From cf6148328ec16e2e5b99d949fc5e6687d6ef7417 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Tue, 5 Dec 2023 02:29:09 -0800 Subject: [PATCH 06/27] Bundle parsing routine --- impeller/compiler/shader_bundle.cc | 106 ++++++++++++++++------------- impeller/compiler/types.cc | 29 ++++++++ impeller/compiler/types.h | 9 +-- 3 files changed, 91 insertions(+), 53 deletions(-) diff --git a/impeller/compiler/shader_bundle.cc b/impeller/compiler/shader_bundle.cc index 780cc092a99a0..af749c6c9c93a 100644 --- a/impeller/compiler/shader_bundle.cc +++ b/impeller/compiler/shader_bundle.cc @@ -10,68 +10,76 @@ namespace impeller { namespace compiler { -/// Compiles and outputs a -bool GenerateShaderBundle(Switches& switches) { - auto json = nlohmann::json::parse(switches.iplr_bundle); +static std::optional ParseShaderBundleConfig( + std::string& json_config) { + auto json = nlohmann::json::parse(json_config); if (!json.is_object()) { std::cerr << "The shader bundle must be a JSON object." << std::endl; - return false; + return std::nullopt; } - /// Bundle parsing. - - IPLRBundleEntries bundle_entries_result; - for (auto& [bundle_name, bundle_value] : json.items()) { - if (!bundle_value.is_object()) { - std::cerr << "Invalid bundle entry \"" << bundle_name + ShaderBundleConfig bundle; + for (auto& [shader_name, shader_value] : json.items()) { + if (bundle.find(shader_name) == bundle.end()) { + std::cerr << "Duplicate shader \"" << shader_name << "\"." << std::endl; + return std::nullopt; + } + if (!shader_value.is_object()) { + std::cerr << "Invalid shader entry \"" << shader_name << "\": Entry is not a JSON object." << std::endl; - return false; + return std::nullopt; } - /// Shader parsing. + ShaderConfig shader; - IPLRBundleEntry bundle_entry_result; - for (auto& [shader_name, shader_value] : bundle_value.items()) { - if (bundle_entry_result.find(shader_name) == bundle_entry_result.end()) { - std::cerr << "Duplicate shader \"" << shader_name << "\" in bundle \"" - << bundle_name << "\"." << std::endl; - return false; - } - if (!bundle_value.is_object()) { - std::cerr << "Invalid shader entry \"" << shader_name - << "\" in bundle \"" << bundle_name - << "\": Entry is not a JSON object." << std::endl; - return false; - } + if (!shader_value.contains("file")) { + std::cerr << "Invalid shader entry \"" << shader_name + << "\": Missing required \"file\" field. \"" << std::endl; + return std::nullopt; + } + shader.source_file_name = shader_value["file"]; + + if (!shader_value.contains("type")) { + std::cerr << "Invalid shader entry \"" << shader_name + << "\": Missing required \"type\" field. \"" << std::endl; + return std::nullopt; + } + shader.type = shader_value["type"]; + if (shader.type == SourceType::kUnknown) { + std::cerr << "Invalid shader entry \"" << shader_name + << "\": Shader type \"" << shader_value["type"] + << "\" is unknown. \"" << std::endl; + return std::nullopt; + } - IPLRShaderEntry stage_result; + shader.language = shader_value.contains("language") + ? ToSourceLanguage(shader_value["language"]) + : SourceLanguage::kGLSL; + if (shader.language == SourceLanguage::kUnknown) { + std::cerr << "Invalid shader entry \"" << shader_name + << "\": Unknown language type \"" << shader_value["language"] + << "\"." << std::endl; + return std::nullopt; + } - stage_result.language = shader_value.contains("language") - ? ToSourceLanguage(shader_value["language"]) - : SourceLanguage::kGLSL; - if (stage_result.language == SourceLanguage::kUnknown) { - std::cerr << "Invalid shader entry \"" << shader_name - << "\" in bundle \"" << bundle_name - << "\": Unknown language type \"" << shader_value["language"] - << "\"." << std::endl; - return false; - } + shader.entry_point = shader_value.contains("entry_point") + ? shader_value["entry_point"] + : "main"; - if (!shader_value.contains("name")) { - std::cerr << "Duplicate IPLR bundle stage entry \"" << shader_name - << "\" in bundle \"" << bundle_name << "\": Entry ." - << std::endl; - return false; - } - stage_result.source_file_name = shader_value["name"]; + bundle[shader_name] = shader; + } - stage_result.entry_point = shader_value.contains("entry_point") - ? shader_value["entry_point"] - : "main"; + return bundle; +} - bundle_entry_result[shader_name] = stage_result; - } - bundle_entries_result[bundle_name] = bundle_entry_result; +/// Parses the given JSON shader bundle configuration and invokes the compiler +/// multiple times to produce a shader bundle file. +bool GenerateShaderBundle(Switches& switches) { + /// Parse the bundle files. + std::optional bundle_config = + ParseShaderBundleConfig(switches.iplr_bundle); + if (!bundle_config) { + return false; } return true; diff --git a/impeller/compiler/types.cc b/impeller/compiler/types.cc index f492471f40d27..5f8347212e369 100644 --- a/impeller/compiler/types.cc +++ b/impeller/compiler/types.cc @@ -4,6 +4,7 @@ #include "impeller/compiler/types.h" +#include #include #include @@ -42,6 +43,34 @@ SourceType SourceTypeFromFileName(const std::string& file_name) { return SourceType::kUnknown; } +SourceType SourceTypeFromString(std::string name) { + for (auto it = name.begin(); it != name.end(); it++) { + *it = std::tolower(static_cast(*it)); + } + + if (name == "vertex") { + return SourceType::kVertexShader; + } + + if (name == "fragment") { + return SourceType::kFragmentShader; + } + + if (name == "tessellationcontrol") { + return SourceType::kTessellationControlShader; + } + + if (name == "tessellationevaluation") { + return SourceType::kTessellationEvaluationShader; + } + + if (name == "compute") { + return SourceType::kComputeShader; + } + + return SourceType::kUnknown; +} + SourceLanguage ToSourceLanguage(const std::string& source_language) { if (source_language == "glsl") { return SourceLanguage::kGLSL; diff --git a/impeller/compiler/types.h b/impeller/compiler/types.h index 7cf3b0adf1640..8fb201ee46fe2 100644 --- a/impeller/compiler/types.h +++ b/impeller/compiler/types.h @@ -43,16 +43,15 @@ enum class SourceLanguage { kHLSL, }; -struct IPLRShaderEntry { +/// A shader config parsed as part of a ShaderBundleConfig. +struct ShaderConfig { std::string source_file_name; SourceType type; SourceLanguage language; std::string entry_point; }; -using IPLRBundleEntry = std::map; - -using IPLRBundleEntries = std::map; +using ShaderBundleConfig = std::map; bool TargetPlatformIsMetal(TargetPlatform platform); @@ -62,6 +61,8 @@ bool TargetPlatformIsVulkan(TargetPlatform platform); SourceType SourceTypeFromFileName(const std::string& file_name); +SourceType SourceTypeFromString(const std::string& name); + std::string SourceTypeToString(SourceType type); std::string TargetPlatformToString(TargetPlatform platform); From 5243eff1d32d3a063bc11763ae9307019f8289bc Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Tue, 5 Dec 2023 03:52:27 -0800 Subject: [PATCH 07/27] Compile the shaders --- impeller/compiler/impellerc_main.cc | 47 +++++++------ impeller/compiler/runtime_stage_data.cc | 33 ++++++--- impeller/compiler/runtime_stage_data.h | 3 + impeller/compiler/shader_bundle.cc | 92 ++++++++++++++++++++++++- impeller/compiler/shader_bundle.h | 3 +- impeller/shader_bundle/shader.cc | 6 +- impeller/shader_bundle/shader.h | 6 +- impeller/shader_bundle/shader_bundle.cc | 2 +- impeller/shader_bundle/shader_bundle.h | 2 +- 9 files changed, 152 insertions(+), 42 deletions(-) diff --git a/impeller/compiler/impellerc_main.cc b/impeller/compiler/impellerc_main.cc index 68616afd929b7..a916f9a6d7527 100644 --- a/impeller/compiler/impellerc_main.cc +++ b/impeller/compiler/impellerc_main.cc @@ -63,7 +63,9 @@ static bool OutputArtifacts(Compiler& compiler, std::shared_ptr source_file_mapping, SourceOptions& options, Reflector::Options& reflector_options) { + // -------------------------------------------------------------------------- /// 1. Invoke the compiler to generate SkSL if needed. + /// std::shared_ptr sksl_mapping; if (switches.iplr && TargetPlatformBundlesSkSL(switches.target_platform)) { @@ -74,8 +76,10 @@ static bool OutputArtifacts(Compiler& compiler, } } - /// 2. Output the source file. When in IPLR/RuntimeStage mode, this is the - /// IPLR flatbuffer file. + // -------------------------------------------------------------------------- + /// 2. Output the source file. When in IPLR/RuntimeStage mode, output the + /// serialized IPLR flatbuffer. + /// auto sl_file_name = std::filesystem::absolute( std::filesystem::current_path() / switches.sl_file_name); @@ -123,8 +127,10 @@ static bool OutputArtifacts(Compiler& compiler, } } + // -------------------------------------------------------------------------- /// 3. Output shader reflection data. /// May include a JSON file, a C++ header, and/or a C++ TU. + /// if (TargetPlatformNeedsReflection(options.target_platform)) { if (!switches.reflection_json_name.empty()) { @@ -168,7 +174,9 @@ static bool OutputArtifacts(Compiler& compiler, } } + // -------------------------------------------------------------------------- /// 4. Output a depfile. + /// if (!switches.depfile_path.empty()) { std::string result_file; @@ -215,24 +223,7 @@ bool Main(const fml::CommandLine& command_line) { Switches::PrintHelp(std::cerr); return false; } - - if (!switches.iplr_bundle.empty()) { - // Invoke the compiler multiple times to build a shader bundle with the - // given iplr_bundle spec. - return GenerateShaderBundle(switches); - } - - // Invoke the compiler and generate reflection data for a single shader or - // runtime stage IPLR. - - std::shared_ptr source_file_mapping = - fml::FileMapping::CreateReadOnly(switches.source_file_name); - if (!source_file_mapping) { - std::cerr << "Could not open input file." << std::endl; - return false; - } - - SourceOptions options; + SourceOptions options; options.target_platform = switches.target_platform; options.source_language = switches.source_language; if (switches.input_type == SourceType::kUnknown) { @@ -253,6 +244,22 @@ bool Main(const fml::CommandLine& command_line) { options.use_half_textures = switches.use_half_textures; options.require_framebuffer_fetch = switches.require_framebuffer_fetch; + if (!switches.iplr_bundle.empty()) { + // Invoke the compiler multiple times to build a shader bundle with the + // given iplr_bundle spec. + return GenerateShaderBundle(switches, options); + } + + std::shared_ptr source_file_mapping = + fml::FileMapping::CreateReadOnly(switches.source_file_name); + if (!source_file_mapping) { + std::cerr << "Could not open input file." << std::endl; + return false; + } + + // Invoke the compiler and generate reflection data for a single shader or + // runtime stage IPLR. + Reflector::Options reflector_options; reflector_options.target_platform = switches.target_platform; reflector_options.entry_point_name = options.entry_point_name; diff --git a/impeller/compiler/runtime_stage_data.cc b/impeller/compiler/runtime_stage_data.cc index 95b35bcbbc1db..5f2e9c95e2d12 100644 --- a/impeller/compiler/runtime_stage_data.cc +++ b/impeller/compiler/runtime_stage_data.cc @@ -285,35 +285,36 @@ std::shared_ptr RuntimeStageData::CreateJsonMapping() const { json_string->size(), [json_string](auto, auto) {}); } -std::shared_ptr RuntimeStageData::CreateMapping() const { +std::unique_ptr RuntimeStageData::CreateFlatbuffer() const { + auto runtime_stage = std::make_unique(); + // The high level object API is used here for writing to the buffer. This is // just a convenience. - fb::RuntimeStageT runtime_stage; - runtime_stage.entrypoint = entrypoint_; + runtime_stage->entrypoint = entrypoint_; const auto stage = ToStage(stage_); if (!stage.has_value()) { VALIDATION_LOG << "Invalid runtime stage."; return nullptr; } - runtime_stage.stage = stage.value(); + runtime_stage->stage = stage.value(); const auto target_platform = ToTargetPlatform(target_platform_); if (!target_platform.has_value()) { VALIDATION_LOG << "Invalid target platform for runtime stage."; return nullptr; } - runtime_stage.target_platform = target_platform.value(); + runtime_stage->target_platform = target_platform.value(); if (!shader_) { VALIDATION_LOG << "No shader specified for runtime stage."; return nullptr; } if (shader_->GetSize() > 0u) { - runtime_stage.shader = {shader_->GetMapping(), - shader_->GetMapping() + shader_->GetSize()}; + runtime_stage->shader = {shader_->GetMapping(), + shader_->GetMapping() + shader_->GetSize()}; } // It is not an error for the SkSL to be ommitted. if (sksl_ && sksl_->GetSize() > 0u) { - runtime_stage.sksl = {sksl_->GetMapping(), - sksl_->GetMapping() + sksl_->GetSize()}; + runtime_stage->sksl = {sksl_->GetMapping(), + sksl_->GetMapping() + sksl_->GetSize()}; } for (const auto& uniform : uniforms_) { auto desc = std::make_unique(); @@ -337,10 +338,20 @@ std::shared_ptr RuntimeStageData::CreateMapping() const { desc->array_elements = uniform.array_elements.value(); } - runtime_stage.uniforms.emplace_back(std::move(desc)); + runtime_stage->uniforms.emplace_back(std::move(desc)); } + + return runtime_stage; +} + +std::shared_ptr RuntimeStageData::CreateMapping() const { + auto runtime_stage = CreateFlatbuffer(); + if (!runtime_stage) { + return nullptr; + } + auto builder = std::make_shared(); - builder->Finish(fb::RuntimeStage::Pack(*builder.get(), &runtime_stage), + builder->Finish(fb::RuntimeStage::Pack(*builder.get(), runtime_stage.get()), fb::RuntimeStageIdentifier()); return std::make_shared(builder->GetBufferPointer(), builder->GetSize(), diff --git a/impeller/compiler/runtime_stage_data.h b/impeller/compiler/runtime_stage_data.h index 45e3476fc84f5..a50f58cf1b020 100644 --- a/impeller/compiler/runtime_stage_data.h +++ b/impeller/compiler/runtime_stage_data.h @@ -10,6 +10,7 @@ #include "flutter/fml/macros.h" #include "flutter/fml/mapping.h" #include "impeller/compiler/types.h" +#include "runtime_stage_types_flatbuffers.h" #include "spirv_parser.hpp" namespace impeller { @@ -39,6 +40,8 @@ class RuntimeStageData { void SetSkSLData(std::shared_ptr sksl); + std::unique_ptr CreateFlatbuffer() const; + std::shared_ptr CreateMapping() const; std::shared_ptr CreateJsonMapping() const; diff --git a/impeller/compiler/shader_bundle.cc b/impeller/compiler/shader_bundle.cc index af749c6c9c93a..b8465d698de14 100644 --- a/impeller/compiler/shader_bundle.cc +++ b/impeller/compiler/shader_bundle.cc @@ -3,8 +3,13 @@ // found in the LICENSE file. #include "impeller/compiler/shader_bundle.h" +#include "impeller/compiler/compiler.h" +#include "impeller/compiler/reflector.h" +#include "impeller/compiler/source_options.h" #include "impeller/compiler/types.h" +#include "impeller/shader_bundle/shader_bundle_flatbuffers.h" +#include "runtime_stage_types_flatbuffers.h" #include "third_party/json/include/nlohmann/json.hpp" namespace impeller { @@ -72,16 +77,99 @@ static std::optional ParseShaderBundleConfig( return bundle; } +static std::unique_ptr GenerateShaderFB( + SourceOptions& options, + const std::string& shader_name, + const ShaderConfig& shader_config) { + auto result = std::make_unique(); + result->name = shader_name; + + std::shared_ptr source_file_mapping = + fml::FileMapping::CreateReadOnly(shader_config.source_file_name); + if (!source_file_mapping) { + std::cerr << "Could not open file for bundled shader \"" << shader_name + << "\"." << std::endl; + return nullptr; + } + + /// Override options. + options.entry_point_name = shader_config.entry_point; + options.type = shader_config.type; + options.source_language = shader_config.language; + + Reflector::Options reflector_options; + reflector_options.target_platform = options.target_platform; + reflector_options.entry_point_name = options.entry_point_name; + reflector_options.shader_name = shader_name; + + Compiler compiler(source_file_mapping, options, reflector_options); + if (!compiler.IsValid()) { + std::cerr << "Compilation failed for bundled shader \"" << shader_name + << "\"." << std::endl; + std::cerr << compiler.GetErrorMessages() << std::endl; + return nullptr; + } + + auto reflector = compiler.GetReflector(); + if (reflector == nullptr) { + std::cerr << "Could not create reflector for bundled shader \"" + << shader_name << "\"." << std::endl; + return nullptr; + } + + auto stage_data = reflector->GetRuntimeStageData(); + if (!stage_data) { + std::cerr << "Runtime stage information was nil for bundled shader \"" + << shader_name << "\"." << std::endl; + return nullptr; + } + + result->shader = stage_data->CreateFlatbuffer(); + if (!result->shader) { + std::cerr << "Failed to create flatbuffer for bundled shader \"" + << shader_name << "\"." << std::endl; + return nullptr; + } + + // TODO(bdero): Vertex attributes. + + return result; +} + /// Parses the given JSON shader bundle configuration and invokes the compiler /// multiple times to produce a shader bundle file. -bool GenerateShaderBundle(Switches& switches) { - /// Parse the bundle files. +bool GenerateShaderBundle(Switches& switches, SourceOptions& options) { + // -------------------------------------------------------------------------- + /// 1. Parse the bundle configuration. + /// + std::optional bundle_config = ParseShaderBundleConfig(switches.iplr_bundle); if (!bundle_config) { return false; } + // -------------------------------------------------------------------------- + /// 2. Build the deserialized shader bundle. + /// + + fb::ShaderBundleT shader_bundle; + + for (const auto& [shader_name, shader_config] : bundle_config.value()) { + std::unique_ptr shader = + GenerateShaderFB(options, shader_name, shader_config); + if (!shader) { + return false; + } + shader_bundle.shaders.push_back(shader); + } + + // -------------------------------------------------------------------------- + /// 3. Serialize the shader bundle and write to disk. + /// + + // TODO(bdero) + return true; } diff --git a/impeller/compiler/shader_bundle.h b/impeller/compiler/shader_bundle.h index d364561c2184b..44be41f4ec29a 100644 --- a/impeller/compiler/shader_bundle.h +++ b/impeller/compiler/shader_bundle.h @@ -2,12 +2,13 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +#include "impeller/compiler/source_options.h" #include "impeller/compiler/switches.h" namespace impeller { namespace compiler { -bool GenerateShaderBundle(Switches& switches); +bool GenerateShaderBundle(Switches& switches, SourceOptions& options); } // namespace compiler } // namespace impeller diff --git a/impeller/shader_bundle/shader.cc b/impeller/shader_bundle/shader.cc index 6194833c59194..67b47aefd2589 100644 --- a/impeller/shader_bundle/shader.cc +++ b/impeller/shader_bundle/shader.cc @@ -6,15 +6,15 @@ namespace impeller { -Shader::Shader() = default; +BundledShader::BundledShader() = default; -Shader::Shader(const fb::Shader* shader) +BundledShader::BundledShader(const fb::Shader* shader) : runtime_stage_( std::make_shared(RuntimeStage(shader->shader()))) { is_valid_ = runtime_stage_->IsValid(); } -bool Shader::IsValid() const { +bool BundledShader::IsValid() const { return is_valid_; } diff --git a/impeller/shader_bundle/shader.h b/impeller/shader_bundle/shader.h index ad639a8360c7a..d446012c882e6 100644 --- a/impeller/shader_bundle/shader.h +++ b/impeller/shader_bundle/shader.h @@ -15,11 +15,11 @@ namespace impeller { class ShaderBundle; -class Shader { +class BundledShader { public: // Note: Default constructor and copy operations required for map usage in // ShaderBundle. - Shader(); + BundledShader(); bool IsValid() const; @@ -28,7 +28,7 @@ class Shader { std::shared_ptr runtime_stage_; - explicit Shader(const fb::Shader* shader); + explicit BundledShader(const fb::Shader* shader); friend ShaderBundle; }; diff --git a/impeller/shader_bundle/shader_bundle.cc b/impeller/shader_bundle/shader_bundle.cc index 1233898fdb737..6a7faa4e7fe57 100644 --- a/impeller/shader_bundle/shader_bundle.cc +++ b/impeller/shader_bundle/shader_bundle.cc @@ -24,7 +24,7 @@ ShaderBundle::ShaderBundle(std::shared_ptr payload) auto* shaders = shader_bundle->shaders(); for (size_t i = 0; i < shaders->size(); i++) { const fb::Shader* shader = shaders->Get(i); - shaders_[shader->name()->str()] = Shader(shader); + shaders_[shader->name()->str()] = BundledShader(shader); if (!shaders_[shader->name()->str()].IsValid()) { return; } diff --git a/impeller/shader_bundle/shader_bundle.h b/impeller/shader_bundle/shader_bundle.h index f48a33bf2a674..6decefab2fd36 100644 --- a/impeller/shader_bundle/shader_bundle.h +++ b/impeller/shader_bundle/shader_bundle.h @@ -26,7 +26,7 @@ class ShaderBundle { bool is_valid_; std::shared_ptr payload_; - std::map shaders_; + std::map shaders_; ShaderBundle(const ShaderBundle&) = delete; ShaderBundle& operator=(const ShaderBundle&) = delete; From 7f4504426446adac8fcf20d0b3d45ddeb07926c0 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Tue, 5 Dec 2023 05:29:38 -0800 Subject: [PATCH 08/27] Write flatbuffer --- impeller/compiler/impellerc_main.cc | 17 +---------------- impeller/compiler/shader_bundle.cc | 26 ++++++++++++++++++++++++-- impeller/compiler/utilities.cc | 15 +++++++++++++++ impeller/compiler/utilities.h | 3 +++ 4 files changed, 43 insertions(+), 18 deletions(-) diff --git a/impeller/compiler/impellerc_main.cc b/impeller/compiler/impellerc_main.cc index a916f9a6d7527..4a174f581f1cd 100644 --- a/impeller/compiler/impellerc_main.cc +++ b/impeller/compiler/impellerc_main.cc @@ -19,21 +19,6 @@ namespace impeller { namespace compiler { -// Sets the file access mode of the file at path 'p' to 0644. -static bool SetPermissiveAccess(const std::filesystem::path& p) { - auto permissions = - std::filesystem::perms::owner_read | std::filesystem::perms::owner_write | - std::filesystem::perms::group_read | std::filesystem::perms::others_read; - std::error_code error; - std::filesystem::permissions(p, permissions, error); - if (error) { - std::cerr << "Failed to set access on file '" << p - << "': " << error.message() << std::endl; - return false; - } - return true; -} - /// Run the shader compiler to geneate SkSL. /// If there is an error, prints error text and returns `nullptr`. static std::shared_ptr CompileSkSL( @@ -223,7 +208,7 @@ bool Main(const fml::CommandLine& command_line) { Switches::PrintHelp(std::cerr); return false; } - SourceOptions options; + SourceOptions options; options.target_platform = switches.target_platform; options.source_language = switches.source_language; if (switches.input_type == SourceType::kUnknown) { diff --git a/impeller/compiler/shader_bundle.cc b/impeller/compiler/shader_bundle.cc index b8465d698de14..b224796fa55aa 100644 --- a/impeller/compiler/shader_bundle.cc +++ b/impeller/compiler/shader_bundle.cc @@ -8,8 +8,8 @@ #include "impeller/compiler/source_options.h" #include "impeller/compiler/types.h" +#include "impeller/compiler/utilities.h" #include "impeller/shader_bundle/shader_bundle_flatbuffers.h" -#include "runtime_stage_types_flatbuffers.h" #include "third_party/json/include/nlohmann/json.hpp" namespace impeller { @@ -168,7 +168,29 @@ bool GenerateShaderBundle(Switches& switches, SourceOptions& options) { /// 3. Serialize the shader bundle and write to disk. /// - // TODO(bdero) + auto builder = std::make_shared(); + builder->Finish(fb::ShaderBundle::Pack(*builder.get(), &shader_bundle), + fb::ShaderBundleIdentifier()); + auto mapping = std::make_shared( + builder->GetBufferPointer(), builder->GetSize(), + [builder](auto, auto) {}); + + auto sl_file_name = std::filesystem::absolute( + std::filesystem::current_path() / switches.sl_file_name); + + if (!fml::WriteAtomically(*switches.working_directory, // + Utf8FromPath(sl_file_name).c_str(), // + *mapping // + )) { + std::cerr << "Could not write file to " << switches.sl_file_name + << std::endl; + return false; + } + // Tools that consume the runtime stage data expect the access mode to + // be 0644. + if (!SetPermissiveAccess(sl_file_name)) { + return false; + } return true; } diff --git a/impeller/compiler/utilities.cc b/impeller/compiler/utilities.cc index 75a535a0c7092..8de72a1b24765 100644 --- a/impeller/compiler/utilities.cc +++ b/impeller/compiler/utilities.cc @@ -6,11 +6,26 @@ #include #include +#include #include namespace impeller { namespace compiler { +bool SetPermissiveAccess(const std::filesystem::path& p) { + auto permissions = + std::filesystem::perms::owner_read | std::filesystem::perms::owner_write | + std::filesystem::perms::group_read | std::filesystem::perms::others_read; + std::error_code error; + std::filesystem::permissions(p, permissions, error); + if (error) { + std::cerr << "Failed to set access on file '" << p + << "': " << error.message() << std::endl; + return false; + } + return true; +} + std::string Utf8FromPath(const std::filesystem::path& path) { return reinterpret_cast(path.u8string().c_str()); } diff --git a/impeller/compiler/utilities.h b/impeller/compiler/utilities.h index d1d27436ac08e..ba1145f128521 100644 --- a/impeller/compiler/utilities.h +++ b/impeller/compiler/utilities.h @@ -11,6 +11,9 @@ namespace impeller { namespace compiler { +/// @brief Sets the file access mode of the file at path 'p' to 0644. +bool SetPermissiveAccess(const std::filesystem::path& p); + /// @brief Converts a native format path to a utf8 string. /// /// This utility uses `path::u8string()` to convert native paths to From 35614d4ca8a9a4c4ed111aa2c4937d00e860441d Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Wed, 6 Dec 2023 11:38:13 -0800 Subject: [PATCH 09/27] Runtime crash fix --- impeller/compiler/shader_bundle.cc | 6 +++--- impeller/compiler/types.h | 2 +- impeller/fixtures/BUILD.gn | 12 +++++++----- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/impeller/compiler/shader_bundle.cc b/impeller/compiler/shader_bundle.cc index b224796fa55aa..96408e7744ec0 100644 --- a/impeller/compiler/shader_bundle.cc +++ b/impeller/compiler/shader_bundle.cc @@ -25,7 +25,7 @@ static std::optional ParseShaderBundleConfig( ShaderBundleConfig bundle; for (auto& [shader_name, shader_value] : json.items()) { - if (bundle.find(shader_name) == bundle.end()) { + if (bundle.find(shader_name) != bundle.end()) { std::cerr << "Duplicate shader \"" << shader_name << "\"." << std::endl; return std::nullopt; } @@ -49,7 +49,7 @@ static std::optional ParseShaderBundleConfig( << "\": Missing required \"type\" field. \"" << std::endl; return std::nullopt; } - shader.type = shader_value["type"]; + shader.type = SourceTypeFromString(shader_value["type"]); if (shader.type == SourceType::kUnknown) { std::cerr << "Invalid shader entry \"" << shader_name << "\": Shader type \"" << shader_value["type"] @@ -161,7 +161,7 @@ bool GenerateShaderBundle(Switches& switches, SourceOptions& options) { if (!shader) { return false; } - shader_bundle.shaders.push_back(shader); + shader_bundle.shaders.push_back(std::move(shader)); } // -------------------------------------------------------------------------- diff --git a/impeller/compiler/types.h b/impeller/compiler/types.h index 8fb201ee46fe2..2f14cc701a607 100644 --- a/impeller/compiler/types.h +++ b/impeller/compiler/types.h @@ -61,7 +61,7 @@ bool TargetPlatformIsVulkan(TargetPlatform platform); SourceType SourceTypeFromFileName(const std::string& file_name); -SourceType SourceTypeFromString(const std::string& name); +SourceType SourceTypeFromString(std::string name); std::string SourceTypeToString(SourceType type); diff --git a/impeller/fixtures/BUILD.gn b/impeller/fixtures/BUILD.gn index 073e3bee59f32..dadeb70982695 100644 --- a/impeller/fixtures/BUILD.gn +++ b/impeller/fixtures/BUILD.gn @@ -121,17 +121,19 @@ impellerc("flutter_gpu_shaders") { "flutter_gpu_texture.frag", "flutter_gpu_texture.vert", ] - sl_file_extension = "iplr" + sl_file_extension = "shaderbundle" shader_target_flag = "--runtime-stage-metal" - iplr = true - iplr_bundle = "temporary" + iplr_bundle = "{\"UnlitFragment\": {\"type\": \"fragment\", \"file\": \"../../flutter/impeller/fixtures/flutter_gpu_unlit.frag\"}, \"UnlitVertex\": {\"type\": \"vertex\", \"file\": \"../../flutter/impeller/fixtures/flutter_gpu_unlit.vert\"}, \"TextureFragment\": {\"type\": \"fragment\", \"file\": \"../../flutter/impeller/fixtures/flutter_gpu_texture.frag\"}, \"TextureVertex\": {\"type\": \"vertex\", \"file\": \"../../flutter/impeller/fixtures/flutter_gpu_texture.vert\"}}" } test_fixtures("flutter_gpu_fixtures") { dart_main = "dart_tests.dart" - fixtures = - filter_include(get_target_outputs(":flutter_gpu_shaders"), [ "*.iplr" ]) + fixtures = filter_include(get_target_outputs(":flutter_gpu_shaders"), + [ + "*.iplr", + "*.shaderbundle", + ]) deps = [ ":flutter_gpu_shaders" ] } From 34e063a7c6696b615a9f6345a65d99d89bc4deb9 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Fri, 8 Dec 2023 11:50:26 -0800 Subject: [PATCH 10/27] impellerc GN rules --- impeller/compiler/impellerc_main.cc | 4 +- impeller/compiler/shader_bundle.cc | 2 +- impeller/compiler/switches.cc | 28 ++++-- impeller/compiler/switches.h | 5 +- impeller/compiler/types.cc | 8 -- impeller/fixtures/BUILD.gn | 4 +- impeller/tools/impeller.gni | 139 +++++++++++++++++++++------- 7 files changed, 130 insertions(+), 60 deletions(-) diff --git a/impeller/compiler/impellerc_main.cc b/impeller/compiler/impellerc_main.cc index 4a174f581f1cd..48c7e9cc5fc14 100644 --- a/impeller/compiler/impellerc_main.cc +++ b/impeller/compiler/impellerc_main.cc @@ -229,9 +229,9 @@ bool Main(const fml::CommandLine& command_line) { options.use_half_textures = switches.use_half_textures; options.require_framebuffer_fetch = switches.require_framebuffer_fetch; - if (!switches.iplr_bundle.empty()) { + if (!switches.shader_bundle.empty()) { // Invoke the compiler multiple times to build a shader bundle with the - // given iplr_bundle spec. + // given shader_bundle spec. return GenerateShaderBundle(switches, options); } diff --git a/impeller/compiler/shader_bundle.cc b/impeller/compiler/shader_bundle.cc index 96408e7744ec0..704e469a80a02 100644 --- a/impeller/compiler/shader_bundle.cc +++ b/impeller/compiler/shader_bundle.cc @@ -144,7 +144,7 @@ bool GenerateShaderBundle(Switches& switches, SourceOptions& options) { /// std::optional bundle_config = - ParseShaderBundleConfig(switches.iplr_bundle); + ParseShaderBundleConfig(switches.shader_bundle); if (!bundle_config) { return false; } diff --git a/impeller/compiler/switches.cc b/impeller/compiler/switches.cc index 71299e5e89157..dbaf911df17b0 100644 --- a/impeller/compiler/switches.cc +++ b/impeller/compiler/switches.cc @@ -53,7 +53,8 @@ void Switches::PrintHelp(std::ostream& stream) { } stream << "}" << std::endl; stream << "--sl=" << std::endl; - stream << "--spirv=" << std::endl; + stream << "--spirv= (ignored for --shader-bundle)" + << std::endl; stream << "[optional] --source-language=glsl|hlsl (default: glsl)" << std::endl; stream << "[optional] --entry-point= (default: main; " @@ -61,8 +62,8 @@ void Switches::PrintHelp(std::ostream& stream) { << std::endl; stream << "[optional] --iplr (causes --sl file to be emitted in iplr format)" << std::endl; - stream << "[optional] --iplr-bundle= (causes --sl file to be " - "emitted in the iplr bundle format)" + stream << "[optional] --shader-bundle= (causes --sl file to be " + "emitted in Flutter GPU's shader bundle format)" << std::endl; stream << "[optional] --reflection-json=" << std::endl; stream << "[optional] --reflection-header=" @@ -122,7 +123,8 @@ Switches::Switches(const fml::CommandLine& command_line) input_type(SourceTypeFromCommandLine(command_line)), sl_file_name(command_line.GetOptionValueWithDefault("sl", "")), iplr(command_line.HasOption("iplr")), - iplr_bundle(command_line.GetOptionValueWithDefault("iplr-bundle", "")), + shader_bundle( + command_line.GetOptionValueWithDefault("shader-bundle", "")), spirv_file_name(command_line.GetOptionValueWithDefault("spirv", "")), reflection_json_name( command_line.GetOptionValueWithDefault("reflection-json", "")), @@ -193,6 +195,12 @@ Switches::Switches(const fml::CommandLine& command_line) } bool Switches::AreValid(std::ostream& explain) const { + // When producing a shader bundle, all flags related to single shader inputs + // and outputs such as `--input` and `--spirv-file-name` are ignored. Instead, + // input files are read from the shader bundle spec and a single flatbuffer + // containing all compiled shaders and reflection state is output to `--sl`. + const bool shader_bundle_mode = !shader_bundle.empty(); + bool valid = true; if (target_platform == TargetPlatform::kUnknown) { explain << "The target platform (only one) was not specified." << std::endl; @@ -211,7 +219,7 @@ bool Switches::AreValid(std::ostream& explain) const { valid = false; } - if (source_file_name.empty()) { + if (source_file_name.empty() && !shader_bundle_mode) { explain << "Input file name was empty." << std::endl; valid = false; } @@ -221,15 +229,15 @@ bool Switches::AreValid(std::ostream& explain) const { valid = false; } - if (spirv_file_name.empty()) { + if (spirv_file_name.empty() && !shader_bundle_mode) { explain << "Spirv file name was empty." << std::endl; valid = false; } - if (iplr && !iplr_bundle.empty()) { - explain - << "--iplr and --iplr-bundle flag cannot be specified at the same time" - << std::endl; + if (iplr && shader_bundle_mode) { + explain << "--iplr and --shader-bundle flag cannot be specified at the " + "same time" + << std::endl; valid = false; } diff --git a/impeller/compiler/switches.h b/impeller/compiler/switches.h index c02e344fa4a88..7ac6e0c540a7d 100644 --- a/impeller/compiler/switches.h +++ b/impeller/compiler/switches.h @@ -22,9 +22,12 @@ struct Switches { std::vector include_directories = {}; std::string source_file_name = ""; SourceType input_type = SourceType::kUnknown; + /// The raw shader file output by the compiler. For --iplr and + /// --shader-bundle modes, this is used as the filename for the output + /// flatbuffer output. std::string sl_file_name = ""; bool iplr = false; - std::string iplr_bundle = ""; + std::string shader_bundle = ""; std::string spirv_file_name = ""; std::string reflection_json_name = ""; std::string reflection_header_name = ""; diff --git a/impeller/compiler/types.cc b/impeller/compiler/types.cc index 5f8347212e369..685122bfb8ae8 100644 --- a/impeller/compiler/types.cc +++ b/impeller/compiler/types.cc @@ -56,14 +56,6 @@ SourceType SourceTypeFromString(std::string name) { return SourceType::kFragmentShader; } - if (name == "tessellationcontrol") { - return SourceType::kTessellationControlShader; - } - - if (name == "tessellationevaluation") { - return SourceType::kTessellationEvaluationShader; - } - if (name == "compute") { return SourceType::kComputeShader; } diff --git a/impeller/fixtures/BUILD.gn b/impeller/fixtures/BUILD.gn index dadeb70982695..4c6fbcb046f2f 100644 --- a/impeller/fixtures/BUILD.gn +++ b/impeller/fixtures/BUILD.gn @@ -121,9 +121,9 @@ impellerc("flutter_gpu_shaders") { "flutter_gpu_texture.frag", "flutter_gpu_texture.vert", ] - sl_file_extension = "shaderbundle" shader_target_flag = "--runtime-stage-metal" - iplr_bundle = "{\"UnlitFragment\": {\"type\": \"fragment\", \"file\": \"../../flutter/impeller/fixtures/flutter_gpu_unlit.frag\"}, \"UnlitVertex\": {\"type\": \"vertex\", \"file\": \"../../flutter/impeller/fixtures/flutter_gpu_unlit.vert\"}, \"TextureFragment\": {\"type\": \"fragment\", \"file\": \"../../flutter/impeller/fixtures/flutter_gpu_texture.frag\"}, \"TextureVertex\": {\"type\": \"vertex\", \"file\": \"../../flutter/impeller/fixtures/flutter_gpu_texture.vert\"}}" + shader_bundle = "{\"UnlitFragment\": {\"type\": \"fragment\", \"file\": \"../../flutter/impeller/fixtures/flutter_gpu_unlit.frag\"}, \"UnlitVertex\": {\"type\": \"vertex\", \"file\": \"../../flutter/impeller/fixtures/flutter_gpu_unlit.vert\"}, \"TextureFragment\": {\"type\": \"fragment\", \"file\": \"../../flutter/impeller/fixtures/flutter_gpu_texture.frag\"}, \"TextureVertex\": {\"type\": \"vertex\", \"file\": \"../../flutter/impeller/fixtures/flutter_gpu_texture.vert\"}}" + shader_bundle_output = "playground.shaderbundle" } test_fixtures("flutter_gpu_fixtures") { diff --git a/impeller/tools/impeller.gni b/impeller/tools/impeller.gni index 4d48704493975..7e9d8c06e70ee 100644 --- a/impeller/tools/impeller.gni +++ b/impeller/tools/impeller.gni @@ -242,18 +242,35 @@ template("embed_blob") { # the impeller_use_prebuilt_impellerc argument. Forwards all variables to # compiled_action_foreach or action_foreach as appropriate. template("_impellerc") { - if (impeller_use_prebuilt_impellerc == "") { - compiled_action_foreach(target_name) { - forward_variables_from(invoker, "*") - tool = "//flutter/impeller/compiler:impellerc" + if (invoker.single_invocation) { + if (impeller_use_prebuilt_impellerc == "") { + compiled_action(target_name) { + forward_variables_from(invoker, "*") + tool = "//flutter/impeller/compiler:impellerc" + } + } else { + action(target_name) { + forward_variables_from(invoker, "*", [ "args" ]) + script = "//build/gn_run_binary.py" + impellerc_path = + rebase_path(impeller_use_prebuilt_impellerc, root_build_dir) + args = [ impellerc_path ] + invoker.args + } } } else { - action_foreach(target_name) { - forward_variables_from(invoker, "*", [ "args" ]) - script = "//build/gn_run_binary.py" - impellerc_path = - rebase_path(impeller_use_prebuilt_impellerc, root_build_dir) - args = [ impellerc_path ] + invoker.args + if (impeller_use_prebuilt_impellerc == "") { + compiled_action_foreach(target_name) { + forward_variables_from(invoker, "*") + tool = "//flutter/impeller/compiler:impellerc" + } + } else { + action_foreach(target_name) { + forward_variables_from(invoker, "*", [ "args" ]) + script = "//build/gn_run_binary.py" + impellerc_path = + rebase_path(impeller_use_prebuilt_impellerc, root_build_dir) + args = [ impellerc_path ] + invoker.args + } } } } @@ -262,8 +279,13 @@ template("impellerc") { assert(defined(invoker.shaders), "Impeller shaders must be specified.") assert(defined(invoker.shader_target_flag), "The flag to impellerc for target selection must be specified.") - assert(defined(invoker.sl_file_extension), + assert(defined(invoker.sl_file_extension) || defined(invoker.shader_bundle), "The extension of the SL file must be specified (metal, glsl, etc..).") + if (defined(invoker.shader_bundle)) { + assert( + defined(invoker.shader_bundle_output), + "When shader_bundle is specified, shader_output_bundle must also be specified.") + } sksl = invoker.shader_target_flag == "--sksl" iplr = false @@ -279,17 +301,26 @@ template("impellerc") { not_needed([ "iplr", "sksl", + "shader_bundle", + "shader_bundle_output", ]) # Optional: invoker.iplr Causes --sl output to be in iplr format. - # Optional: invoker.iplr_bundle specifies a Flutter GPU shader bundle configuration. + # Optional: invoker.shader_bundle specifies a Flutter GPU shader bundle configuration. + # Optional: invoker.shader_bundle_output specifies the output filename of the shader + # bundle. This is required if invoker.shader_bundle is supplied. # Optional: invoker.defines specifies a list of valueless macro definitions. # Optional: invoker.intermediates_subdir specifies the subdirectory in which # to put intermediates. # Optional: invoker.json Causes output format to be JSON instead of flatbuffer. _impellerc(target_name) { - sources = invoker.shaders + shader_bundle = defined(invoker.shader_bundle) + + # When single_invocation is true, impellerc will be invoked exactly once. When it's + # false, impellerc be invoked for each of the source file entries (invoker.shaders). + single_invocation = shader_bundle + if (defined(invoker.intermediates_subdir)) { subdir = invoker.intermediates_subdir generated_dir = "$target_gen_dir/$subdir" @@ -299,22 +330,25 @@ template("impellerc") { shader_target_flag = invoker.shader_target_flag - spirv_intermediate = "$generated_dir/{{source_file_part}}.spirv" - spirv_intermediate_path = rebase_path(spirv_intermediate, root_build_dir) - depfile_path = "$generated_dir/{{source_file_part}}.d" depfile_intermediate_path = rebase_path(depfile_path, root_build_dir) depfile = depfile_path shader_lib_dir = rebase_path("//flutter/impeller/compiler/shader_lib") args = [ - "--input={{source}}", - "--include={{source_dir}}", "--include=$shader_lib_dir", "--depfile=$depfile_intermediate_path", "$shader_target_flag", ] + # When we're in single invocation mode, we can't use source enumeration. + if (!single_invocation) { + args += [ + "--input={{source}}", + "--include={{source_dir}}", + ] + } + if (defined(invoker.gles_language_version)) { gles_language_version = invoker.gles_language_version args += [ "--gles-language-version=$gles_language_version" ] @@ -338,27 +372,60 @@ template("impellerc") { args += [ "--json" ] } - if (sksl) { - sl_intermediate = + if (iplr) { + # When building in IPLR mode, the compiler may be executed twice + args += [ "--iplr" ] + } + + # The `sl_output` is the raw shader file output by the compiler. For --iplr + # and --shader-bundle, this is used as the filename for the flatbuffer to + # output. + + if (shader_bundle) { + # When a shader bundle is specified, don't bother supplying flags for + # the reflection state as these are ignored. In this mode, the compiler + # is invoked multiple times and the reflection state for each shader is + # written to the output flatbuffer. + sl_output = "$generated_dir/${invoker.shader_bundle_output}" + sl_output_path = rebase_path(sl_output, root_build_dir) + + args += [ + "--sl=$sl_output_path", + "--shader-bundle=${invoker.shader_bundle}", + ] + + outputs = [ sl_output ] + } else if (sksl) { + # When SkSL is selected as the `shader_target_flag`, don't generate + # C++ reflection state. Nothing needs to use it and it's likely invalid + # given the special cases when generating SkSL. + # Note that this configuration is orthogonal to the "--iplr" flag + sl_output = "$generated_dir/{{source_file_part}}.${invoker.sl_file_extension}" - sl_intermediate_path = rebase_path(sl_intermediate, root_build_dir) + sl_output_path = rebase_path(sl_output, root_build_dir) + + spirv_intermediate = "$generated_dir/{{source_file_part}}.spirv" + spirv_intermediate_path = rebase_path(spirv_intermediate, root_build_dir) args += [ - "--sl=$sl_intermediate_path", + "--sl=$sl_output_path", "--spirv=$spirv_intermediate_path", ] - if (iplr) { - args += [ "--iplr" ] - } - outputs = [ sl_intermediate ] + outputs = [ sl_output ] } else { - sl_intermediate = + # The default branch. Here we just generate one shader along with all of + # its C++ reflection state. + + sl_output = "$generated_dir/{{source_file_part}}.${invoker.sl_file_extension}" + sl_output_path = rebase_path(sl_output, root_build_dir) + reflection_json_intermediate = "$generated_dir/{{source_file_part}}.json" reflection_header_intermediate = "$generated_dir/{{source_file_part}}.h" reflection_cc_intermediate = "$generated_dir/{{source_file_part}}.cc" - sl_intermediate_path = rebase_path(sl_intermediate, root_build_dir) + spirv_intermediate = "$generated_dir/{{source_file_part}}.spirv" + spirv_intermediate_path = rebase_path(spirv_intermediate, root_build_dir) reflection_json_path = rebase_path(reflection_json_intermediate, root_build_dir) reflection_header_path = @@ -367,21 +434,15 @@ template("impellerc") { rebase_path(reflection_cc_intermediate, root_build_dir) args += [ - "--sl=$sl_intermediate_path", + "--sl=$sl_output_path", "--spirv=$spirv_intermediate_path", "--reflection-json=$reflection_json_path", "--reflection-header=$reflection_header_path", "--reflection-cc=$reflection_cc_path", ] - if (iplr) { - args += [ "--iplr" ] - } - if (defined(invoker.iplr_bundle)) { - args += [ "--iplr-bundle=${invoker.iplr_bundle}" ] - } outputs = [ - sl_intermediate, + sl_output, reflection_header_intermediate, reflection_cc_intermediate, ] @@ -392,6 +453,12 @@ template("impellerc") { args += [ "--define=$def" ] } } + + if (single_invocation) { + inputs = invoker.shaders + } else { + sources = invoker.shaders + } } } From fea9317407aa2d583a908a12f08fe30e0c87c0be Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Sat, 9 Dec 2023 02:32:25 -0800 Subject: [PATCH 11/27] Reflect stage data --- impeller/compiler/reflector.cc | 31 +++++++- impeller/compiler/runtime_stage_data.cc | 77 ++++++++++++++++++- impeller/compiler/runtime_stage_data.h | 16 ++++ .../runtime_stage/runtime_stage_types.fbs | 32 ++++++++ 4 files changed, 152 insertions(+), 4 deletions(-) diff --git a/impeller/compiler/reflector.cc b/impeller/compiler/reflector.cc index 50903b1b5ce97..0c617a57529f6 100644 --- a/impeller/compiler/reflector.cc +++ b/impeller/compiler/reflector.cc @@ -11,12 +11,10 @@ #include #include -#include "flutter/fml/closure.h" #include "flutter/fml/logging.h" #include "impeller/base/strings.h" #include "impeller/base/validation.h" #include "impeller/compiler/code_gen_template.h" -#include "impeller/compiler/types.h" #include "impeller/compiler/uniform_sorter.h" #include "impeller/compiler/utilities.h" #include "impeller/geometry/half.h" @@ -353,6 +351,35 @@ std::shared_ptr Reflector::GenerateRuntimeStageData() const { uniform_description.array_elements = GetArrayElements(spir_type); data->AddUniformDescription(std::move(uniform_description)); } + + // We only need to worry about reflecting vertex attributes. + if (entrypoints.front().execution_model == spv::ExecutionModelVertex) { + const auto inputs = compiler_->get_shader_resources().stage_inputs; + auto input_offsets = ComputeOffsets(inputs); + for (const auto& input : inputs) { + auto location = compiler_->get_decoration( + input.id, spv::Decoration::DecorationLocation); + std::optional offset = input_offsets[location]; + + const auto type = compiler_->get_type(input.type_id); + + InputDescription input_description; + input_description.name = input.name; + input_description.location = compiler_->get_decoration( + input.id, spv::Decoration::DecorationLocation); + input_description.set = compiler_->get_decoration( + input.id, spv::Decoration::DecorationDescriptorSet); + input_description.binding = compiler_->get_decoration( + input.id, spv::Decoration::DecorationBinding); + input_description.type = type.basetype; + input_description.bit_width = type.width; + input_description.vec_size = type.vecsize; + input_description.columns = type.columns; + input_description.offset = offset.value_or(0u); + data->AddInputDescription(std::move(input_description)); + } + } + return data; } diff --git a/impeller/compiler/runtime_stage_data.cc b/impeller/compiler/runtime_stage_data.cc index 5f2e9c95e2d12..47d6cf35266f0 100644 --- a/impeller/compiler/runtime_stage_data.cc +++ b/impeller/compiler/runtime_stage_data.cc @@ -29,6 +29,10 @@ void RuntimeStageData::AddUniformDescription(UniformDescription uniform) { uniforms_.emplace_back(std::move(uniform)); } +void RuntimeStageData::AddInputDescription(InputDescription input) { + inputs_.emplace_back(std::move(input)); +} + void RuntimeStageData::SetShaderData(std::shared_ptr shader) { shader_ = std::move(shader); } @@ -108,7 +112,7 @@ static std::optional ToJsonTargetPlatform(TargetPlatform platform) { FML_UNREACHABLE(); } -static std::optional ToType( +static std::optional ToUniformType( spirv_cross::SPIRType::BaseType type) { switch (type) { case spirv_cross::SPIRType::Boolean: @@ -152,6 +156,49 @@ static std::optional ToType( } FML_UNREACHABLE(); } +static std::optional ToInputType( + spirv_cross::SPIRType::BaseType type) { + switch (type) { + case spirv_cross::SPIRType::Boolean: + return fb::InputDataType::kBoolean; + case spirv_cross::SPIRType::SByte: + return fb::InputDataType::kSignedByte; + case spirv_cross::SPIRType::UByte: + return fb::InputDataType::kUnsignedByte; + case spirv_cross::SPIRType::Short: + return fb::InputDataType::kSignedShort; + case spirv_cross::SPIRType::UShort: + return fb::InputDataType::kUnsignedShort; + case spirv_cross::SPIRType::Int: + return fb::InputDataType::kSignedInt; + case spirv_cross::SPIRType::UInt: + return fb::InputDataType::kUnsignedInt; + case spirv_cross::SPIRType::Int64: + return fb::InputDataType::kSignedInt64; + case spirv_cross::SPIRType::UInt64: + return fb::InputDataType::kUnsignedInt64; + case spirv_cross::SPIRType::Half: + return fb::InputDataType::kHalfFloat; + case spirv_cross::SPIRType::Float: + return fb::InputDataType::kFloat; + case spirv_cross::SPIRType::Double: + return fb::InputDataType::kDouble; + case spirv_cross::SPIRType::Unknown: + case spirv_cross::SPIRType::Void: + case spirv_cross::SPIRType::AtomicCounter: + case spirv_cross::SPIRType::Struct: + case spirv_cross::SPIRType::Image: + case spirv_cross::SPIRType::SampledImage: + case spirv_cross::SPIRType::Sampler: + case spirv_cross::SPIRType::AccelerationStructure: + case spirv_cross::SPIRType::RayQuery: + case spirv_cross::SPIRType::ControlPointArray: + case spirv_cross::SPIRType::Interpolant: + case spirv_cross::SPIRType::Char: + return std::nullopt; + } + FML_UNREACHABLE(); +} static std::optional ToJsonType( spirv_cross::SPIRType::BaseType type) { @@ -327,7 +374,7 @@ std::unique_ptr RuntimeStageData::CreateFlatbuffer() const { desc->location = uniform.location; desc->rows = uniform.rows; desc->columns = uniform.columns; - auto uniform_type = ToType(uniform.type); + auto uniform_type = ToUniformType(uniform.type); if (!uniform_type.has_value()) { VALIDATION_LOG << "Invalid uniform type for runtime stage."; return nullptr; @@ -341,6 +388,32 @@ std::unique_ptr RuntimeStageData::CreateFlatbuffer() const { runtime_stage->uniforms.emplace_back(std::move(desc)); } + for (const auto& input : inputs_) { + auto desc = std::make_unique(); + + desc->name = input.name; + + if (desc->name.empty()) { + VALIDATION_LOG << "Stage input name cannot be empty."; + return nullptr; + } + desc->location = input.location; + desc->set = input.set; + desc->binding = input.binding; + auto input_type = ToInputType(input.type); + if (!input_type.has_value()) { + VALIDATION_LOG << "Invalid uniform type for runtime stage."; + return nullptr; + } + desc->type = input_type.value(); + desc->bit_width = input.bit_width; + desc->vec_size = input.vec_size; + desc->columns = input.columns; + desc->offset = input.offset; + + runtime_stage->inputs.emplace_back(std::move(desc)); + } + return runtime_stage; } diff --git a/impeller/compiler/runtime_stage_data.h b/impeller/compiler/runtime_stage_data.h index a50f58cf1b020..b012d70ff9e00 100644 --- a/impeller/compiler/runtime_stage_data.h +++ b/impeller/compiler/runtime_stage_data.h @@ -26,6 +26,19 @@ struct UniformDescription { std::optional array_elements = std::nullopt; }; +struct InputDescription { + std::string name; + size_t location; + size_t set; + size_t binding; + spirv_cross::SPIRType::BaseType type = + spirv_cross::SPIRType::BaseType::Unknown; + size_t bit_width; + size_t vec_size; + size_t columns; + size_t offset; +}; + class RuntimeStageData { public: RuntimeStageData(std::string entrypoint, @@ -36,6 +49,8 @@ class RuntimeStageData { void AddUniformDescription(UniformDescription uniform); + void AddInputDescription(InputDescription input); + void SetShaderData(std::shared_ptr shader); void SetSkSLData(std::shared_ptr sksl); @@ -51,6 +66,7 @@ class RuntimeStageData { const spv::ExecutionModel stage_; const TargetPlatform target_platform_; std::vector uniforms_; + std::vector inputs_; std::shared_ptr shader_; std::shared_ptr sksl_; diff --git a/impeller/runtime_stage/runtime_stage_types.fbs b/impeller/runtime_stage/runtime_stage_types.fbs index c1fb96e3edb88..962d73fb578ec 100644 --- a/impeller/runtime_stage/runtime_stage_types.fbs +++ b/impeller/runtime_stage/runtime_stage_types.fbs @@ -17,6 +17,7 @@ enum TargetPlatform:byte { kVulkan, } +// The subset of impeller::ShaderType that may be used for uniform bindings. enum UniformDataType:uint32 { kBoolean, kSignedByte, @@ -43,10 +44,41 @@ table UniformDescription { array_elements: uint64; } +// The subset of impeller::ShaderType that may be used for vertex attributes. +enum InputDataType:uint32 { + kBoolean, + kSignedByte, + kUnsignedByte, + kSignedShort, + kUnsignedShort, + kSignedInt, + kUnsignedInt, + kSignedInt64, + kUnsignedInt64, + kHalfFloat, + kFloat, + kDouble, +} + +// This contains the same attribute reflection data as +// impeller::ShaderStageIOSlot. +table StageInput { + name: string; + location: uint64; + set: uint64; + binding: uint64; + type: InputDataType; + bit_width: uint64; + vec_size: uint64; + columns: uint64; + offset: uint64; +} + table RuntimeStage { stage: Stage; target_platform: TargetPlatform; entrypoint: string; + inputs: [StageInput]; uniforms: [UniformDescription]; shader: [ubyte]; sksl: [ubyte]; From d639c2a307cbbe2322589c86123d20ae78fc7880 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Sat, 9 Dec 2023 08:09:50 -0800 Subject: [PATCH 12/27] Runtime shader lib import --- impeller/compiler/runtime_stage_data.cc | 3 +- impeller/renderer/renderer_dart_unittests.cc | 33 +---- impeller/renderer/vertex_descriptor.cc | 7 + impeller/renderer/vertex_descriptor.h | 3 + .../runtime_stage/runtime_stage_types.fbs | 1 - lib/gpu/BUILD.gn | 1 + lib/gpu/shader_library.cc | 131 +++++++++++++++++- lib/gpu/shader_library.h | 9 +- 8 files changed, 149 insertions(+), 39 deletions(-) diff --git a/impeller/compiler/runtime_stage_data.cc b/impeller/compiler/runtime_stage_data.cc index 47d6cf35266f0..a27517c2369e8 100644 --- a/impeller/compiler/runtime_stage_data.cc +++ b/impeller/compiler/runtime_stage_data.cc @@ -177,14 +177,13 @@ static std::optional ToInputType( return fb::InputDataType::kSignedInt64; case spirv_cross::SPIRType::UInt64: return fb::InputDataType::kUnsignedInt64; - case spirv_cross::SPIRType::Half: - return fb::InputDataType::kHalfFloat; case spirv_cross::SPIRType::Float: return fb::InputDataType::kFloat; case spirv_cross::SPIRType::Double: return fb::InputDataType::kDouble; case spirv_cross::SPIRType::Unknown: case spirv_cross::SPIRType::Void: + case spirv_cross::SPIRType::Half: case spirv_cross::SPIRType::AtomicCounter: case spirv_cross::SPIRType::Struct: case spirv_cross::SPIRType::Image: diff --git a/impeller/renderer/renderer_dart_unittests.cc b/impeller/renderer/renderer_dart_unittests.cc index 7e413346cc9a0..ae9a7804d82e6 100644 --- a/impeller/renderer/renderer_dart_unittests.cc +++ b/impeller/renderer/renderer_dart_unittests.cc @@ -18,7 +18,6 @@ #include "flutter/testing/testing.h" #include "fml/memory/ref_ptr.h" #include "impeller/core/shader_types.h" -#include "impeller/fixtures/flutter_gpu_unlit.vert.h" #include "impeller/playground/playground_test.h" #include "impeller/renderer/render_pass.h" #include "impeller/renderer/vertex_descriptor.h" @@ -30,37 +29,11 @@ namespace impeller { namespace testing { -// This helper is for piggybacking on the RuntimeStage infrastructure for -// testing shaders/pipelines before the full shader bundle importer is finished. -static fml::RefPtr OpenRuntimeStageAsShader( - const std::string& fixture_name, - std::shared_ptr vertex_desc) { - auto fixture = flutter::testing::OpenFixtureAsMapping(fixture_name); - assert(fixture); - RuntimeStage stage(std::move(fixture)); - return flutter::gpu::Shader::Make( - stage.GetEntrypoint(), ToShaderStage(stage.GetShaderStage()), - stage.GetCodeMapping(), stage.GetUniforms(), std::move(vertex_desc)); -} - static void InstantiateTestShaderLibrary() { - flutter::gpu::ShaderLibrary::ShaderMap shaders; - auto vertex_desc = std::make_shared(); - vertex_desc->SetStageInputs( - // TODO(bdero): The stage inputs need to be packed into the flatbuffer. - FlutterGpuUnlitVertexShader::kAllShaderStageInputs, - // TODO(bdero): Make the vertex attribute layout fully configurable. - // When encoding commands, allow for specifying a stride, - // type, and vertex buffer slot for each attribute. - // Provide a way to lookup vertex attribute slot locations by - // name from the shader. - FlutterGpuUnlitVertexShader::kInterleavedBufferLayout); - shaders["UnlitVertex"] = OpenRuntimeStageAsShader( - "flutter_gpu_unlit.vert.iplr", std::move(vertex_desc)); - shaders["UnlitFragment"] = - OpenRuntimeStageAsShader("flutter_gpu_unlit.frag.iplr", nullptr); + auto fixture = + flutter::testing::OpenFixtureAsMapping("playground.shaderbundle"); auto library = - flutter::gpu::ShaderLibrary::MakeFromShaders(std::move(shaders)); + flutter::gpu::ShaderLibrary::MakeFromFlatbuffer(std::move(fixture)); flutter::gpu::ShaderLibrary::SetOverride(library); } diff --git a/impeller/renderer/vertex_descriptor.cc b/impeller/renderer/vertex_descriptor.cc index a16ff1309e94a..1c912308859b4 100644 --- a/impeller/renderer/vertex_descriptor.cc +++ b/impeller/renderer/vertex_descriptor.cc @@ -25,6 +25,13 @@ void VertexDescriptor::SetStageInputs( } } +void VertexDescriptor::SetStageInputs( + const std::vector& inputs, + const std::vector& layout) { + inputs_.insert(inputs_.end(), inputs.begin(), inputs.end()); + layouts_.insert(layouts_.end(), layout.begin(), layout.end()); +} + void VertexDescriptor::RegisterDescriptorSetLayouts( const DescriptorSetLayout desc_set_layout[], size_t count) { diff --git a/impeller/renderer/vertex_descriptor.h b/impeller/renderer/vertex_descriptor.h index 72d12e11df61e..de34cfe8e4324 100644 --- a/impeller/renderer/vertex_descriptor.h +++ b/impeller/renderer/vertex_descriptor.h @@ -38,6 +38,9 @@ class VertexDescriptor final : public Comparable { layout.size()); } + void SetStageInputs(const std::vector& inputs, + const std::vector& layout); + template void RegisterDescriptorSetLayouts( const std::array& inputs) { diff --git a/impeller/runtime_stage/runtime_stage_types.fbs b/impeller/runtime_stage/runtime_stage_types.fbs index 962d73fb578ec..b0f7baf152957 100644 --- a/impeller/runtime_stage/runtime_stage_types.fbs +++ b/impeller/runtime_stage/runtime_stage_types.fbs @@ -55,7 +55,6 @@ enum InputDataType:uint32 { kUnsignedInt, kSignedInt64, kUnsignedInt64, - kHalfFloat, kFloat, kDouble, } diff --git a/lib/gpu/BUILD.gn b/lib/gpu/BUILD.gn index b26dc419dc864..da7da8eeae0b9 100644 --- a/lib/gpu/BUILD.gn +++ b/lib/gpu/BUILD.gn @@ -63,6 +63,7 @@ source_set("gpu") { deps = [ "//flutter/impeller", "//flutter/impeller/display_list:skia_conversions", + "//flutter/impeller/shader_bundle:shader_bundle_flatbuffers", "//flutter/lib/ui", "//flutter/third_party/tonic", ] diff --git a/lib/gpu/shader_library.cc b/lib/gpu/shader_library.cc index c82fbcc5862f3..734d6fe70b032 100644 --- a/lib/gpu/shader_library.cc +++ b/lib/gpu/shader_library.cc @@ -10,8 +10,12 @@ #include "flutter/lib/gpu/shader.h" #include "fml/mapping.h" #include "fml/memory/ref_ptr.h" +#include "impeller/core/runtime_types.h" +#include "impeller/core/shader_types.h" #include "impeller/renderer/vertex_descriptor.h" #include "impeller/runtime_stage/runtime_stage.h" +#include "impeller/shader_bundle/shader_bundle_flatbuffers.h" +#include "runtime_stage_types_flatbuffers.h" namespace flutter { namespace gpu { @@ -85,9 +89,125 @@ fml::RefPtr ShaderLibrary::MakeFromAsset( } fml::RefPtr ShaderLibrary::MakeFromShaders(ShaderMap shaders) { - auto res = - fml::MakeRefCounted(std::move(shaders)); - return res; + return fml::MakeRefCounted(nullptr, + std::move(shaders)); +} + +static impeller::ShaderType FromInputType( + impeller::fb::InputDataType input_type) { + switch (input_type) { + case impeller::fb::InputDataType::kBoolean: + return impeller::ShaderType::kBoolean; + case impeller::fb::InputDataType::kSignedByte: + return impeller::ShaderType::kSignedByte; + case impeller::fb::InputDataType::kUnsignedByte: + return impeller::ShaderType::kUnsignedByte; + case impeller::fb::InputDataType::kSignedShort: + return impeller::ShaderType::kSignedShort; + case impeller::fb::InputDataType::kUnsignedShort: + return impeller::ShaderType::kUnsignedShort; + case impeller::fb::InputDataType::kSignedInt: + return impeller::ShaderType::kSignedInt; + case impeller::fb::InputDataType::kUnsignedInt: + return impeller::ShaderType::kUnsignedInt; + case impeller::fb::InputDataType::kSignedInt64: + return impeller::ShaderType::kSignedInt64; + case impeller::fb::InputDataType::kUnsignedInt64: + return impeller::ShaderType::kUnsignedInt64; + case impeller::fb::InputDataType::kFloat: + return impeller::ShaderType::kFloat; + case impeller::fb::InputDataType::kDouble: + return impeller::ShaderType::kDouble; + } +} + +static size_t SizeOfInputType(impeller::fb::InputDataType input_type) { + switch (input_type) { + case impeller::fb::InputDataType::kBoolean: + return 1; + case impeller::fb::InputDataType::kSignedByte: + return 1; + case impeller::fb::InputDataType::kUnsignedByte: + return 1; + case impeller::fb::InputDataType::kSignedShort: + return 2; + case impeller::fb::InputDataType::kUnsignedShort: + return 2; + case impeller::fb::InputDataType::kSignedInt: + return 4; + case impeller::fb::InputDataType::kUnsignedInt: + return 4; + case impeller::fb::InputDataType::kSignedInt64: + return 8; + case impeller::fb::InputDataType::kUnsignedInt64: + return 8; + case impeller::fb::InputDataType::kFloat: + return 4; + case impeller::fb::InputDataType::kDouble: + return 8; + } +} + +fml::RefPtr ShaderLibrary::MakeFromFlatbuffer( + std::shared_ptr payload) { + if (payload == nullptr || !payload->GetMapping()) { + return nullptr; + } + if (!impeller::fb::ShaderBundleBufferHasIdentifier(payload->GetMapping())) { + return nullptr; + } + auto* bundle = impeller::fb::GetShaderBundle(payload->GetMapping()); + if (!bundle) { + return nullptr; + } + + ShaderLibrary::ShaderMap shader_map; + + for (const auto* bundled_shader : *bundle->shaders()) { + const impeller::fb::RuntimeStage* runtime_stage = bundled_shader->shader(); + impeller::RuntimeStage stage(runtime_stage); + + std::shared_ptr vertex_descriptor = nullptr; + if (stage.GetShaderStage() == impeller::RuntimeShaderStage::kVertex) { + vertex_descriptor = std::make_shared(); + auto inputs_fb = runtime_stage->inputs(); + + std::vector inputs(inputs_fb->size()); + size_t default_stride = 0; + for (const auto& input : *inputs_fb) { + impeller::ShaderStageIOSlot slot; + slot.name = input->name()->c_str(); + slot.location = input->location(); + slot.set = input->set(); + slot.binding = input->binding(); + slot.type = FromInputType(input->type()); + slot.bit_width = input->bit_width(); + slot.vec_size = input->vec_size(); + slot.columns = input->columns(); + slot.offset = input->offset(); + inputs.emplace_back(slot); + + default_stride += + SizeOfInputType(input->type()) * slot.vec_size * slot.columns; + } + std::vector layouts = { + impeller::ShaderStageBufferLayout{ + .stride = default_stride, + .binding = 0u, + }}; + + vertex_descriptor->SetStageInputs(inputs, layouts); + } + + auto shader = flutter::gpu::Shader::Make( + stage.GetEntrypoint(), ToShaderStage(stage.GetShaderStage()), + stage.GetCodeMapping(), stage.GetUniforms(), + std::move(vertex_descriptor)); + shader_map[bundled_shader->name()->str()] = std::move(shader); + } + + return fml::MakeRefCounted( + std::move(payload), std::move(shader_map)); } void ShaderLibrary::SetOverride( @@ -109,8 +229,9 @@ fml::RefPtr ShaderLibrary::GetShader(const std::string& shader_name, return shader; } -ShaderLibrary::ShaderLibrary(ShaderMap shaders) - : shaders_(std::move(shaders)) {} +ShaderLibrary::ShaderLibrary(std::shared_ptr payload, + ShaderMap shaders) + : payload_(std::move(payload)), shaders_(std::move(shaders)) {} ShaderLibrary::~ShaderLibrary() = default; diff --git a/lib/gpu/shader_library.h b/lib/gpu/shader_library.h index 35fd737f7a7a1..92d847b559a8a 100644 --- a/lib/gpu/shader_library.h +++ b/lib/gpu/shader_library.h @@ -4,6 +4,7 @@ #pragma once +#include #include #include @@ -11,6 +12,7 @@ #include "flutter/lib/gpu/shader.h" #include "flutter/lib/ui/dart_wrapper.h" #include "fml/memory/ref_ptr.h" +#include "impeller/shader_bundle/shader_bundle_flatbuffers.h" namespace flutter { namespace gpu { @@ -28,6 +30,9 @@ class ShaderLibrary : public RefCountedDartWrappable { static fml::RefPtr MakeFromShaders(ShaderMap shaders); + static fml::RefPtr MakeFromFlatbuffer( + std::shared_ptr payload); + /// Sets a return override for `MakeFromAsset` for testing purposes. static void SetOverride(fml::RefPtr override_shader_library); @@ -42,9 +47,11 @@ class ShaderLibrary : public RefCountedDartWrappable { /// this library. static fml::RefPtr override_shader_library_; + std::shared_ptr payload_; ShaderMap shaders_; - explicit ShaderLibrary(ShaderMap shaders); + explicit ShaderLibrary(std::shared_ptr payload, + ShaderMap shaders); FML_DISALLOW_COPY_AND_ASSIGN(ShaderLibrary); }; From fe51086f6f86f814abed89747d4c73e7b064f243 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Sat, 9 Dec 2023 23:11:00 -0800 Subject: [PATCH 13/27] Fix entrypoint naming mismatch --- impeller/compiler/shader_bundle.cc | 4 +++- impeller/renderer/vertex_descriptor.h | 1 - lib/gpu/shader_library.cc | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/impeller/compiler/shader_bundle.cc b/impeller/compiler/shader_bundle.cc index 704e469a80a02..425c361ae5661 100644 --- a/impeller/compiler/shader_bundle.cc +++ b/impeller/compiler/shader_bundle.cc @@ -93,9 +93,11 @@ static std::unique_ptr GenerateShaderFB( } /// Override options. - options.entry_point_name = shader_config.entry_point; options.type = shader_config.type; options.source_language = shader_config.language; + options.entry_point_name = EntryPointFunctionNameFromSourceName( + shader_config.source_file_name, options.type, options.source_language, + shader_config.entry_point); Reflector::Options reflector_options; reflector_options.target_platform = options.target_platform; diff --git a/impeller/renderer/vertex_descriptor.h b/impeller/renderer/vertex_descriptor.h index de34cfe8e4324..db7cb7aae61e3 100644 --- a/impeller/renderer/vertex_descriptor.h +++ b/impeller/renderer/vertex_descriptor.h @@ -6,7 +6,6 @@ #include -#include "flutter/fml/macros.h" #include "impeller/base/comparable.h" #include "impeller/core/shader_types.h" diff --git a/lib/gpu/shader_library.cc b/lib/gpu/shader_library.cc index 734d6fe70b032..5dde31add6a5f 100644 --- a/lib/gpu/shader_library.cc +++ b/lib/gpu/shader_library.cc @@ -172,7 +172,8 @@ fml::RefPtr ShaderLibrary::MakeFromFlatbuffer( vertex_descriptor = std::make_shared(); auto inputs_fb = runtime_stage->inputs(); - std::vector inputs(inputs_fb->size()); + std::vector inputs; + inputs.reserve(inputs_fb->size()); size_t default_stride = 0; for (const auto& input : *inputs_fb) { impeller::ShaderStageIOSlot slot; From b475d3558539c9ba7f7573011ba9a693fbd232c8 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Sun, 10 Dec 2023 01:10:38 -0800 Subject: [PATCH 14/27] Implement asset loading --- lib/gpu/shader_library.cc | 73 +++++++++------------------------------ 1 file changed, 16 insertions(+), 57 deletions(-) diff --git a/lib/gpu/shader_library.cc b/lib/gpu/shader_library.cc index 5dde31add6a5f..72a39a138824e 100644 --- a/lib/gpu/shader_library.cc +++ b/lib/gpu/shader_library.cc @@ -6,8 +6,12 @@ #include +#include "flutter/assets/asset_manager.h" +#include "flutter/impeller/runtime_stage/runtime_stage_types_flatbuffers.h" #include "flutter/lib/gpu/fixtures.h" #include "flutter/lib/gpu/shader.h" +#include "flutter/lib/ui/ui_dart_state.h" +#include "flutter/lib/ui/window/platform_configuration.h" #include "fml/mapping.h" #include "fml/memory/ref_ptr.h" #include "impeller/core/runtime_types.h" @@ -15,56 +19,10 @@ #include "impeller/renderer/vertex_descriptor.h" #include "impeller/runtime_stage/runtime_stage.h" #include "impeller/shader_bundle/shader_bundle_flatbuffers.h" -#include "runtime_stage_types_flatbuffers.h" namespace flutter { namespace gpu { -// ===[ BEGIN MEMES ]=========================================================== -static fml::RefPtr OpenRuntimeStageAsShader( - std::shared_ptr payload, - const std::shared_ptr& vertex_desc) { - impeller::RuntimeStage stage(std::move(payload)); - return Shader::Make(stage.GetEntrypoint(), - ToShaderStage(stage.GetShaderStage()), - stage.GetCodeMapping(), stage.GetUniforms(), vertex_desc); -} - -static fml::RefPtr InstantiateTestShaderLibrary() { - ShaderLibrary::ShaderMap shaders; - { - auto vertex_desc = std::make_shared(); - vertex_desc->SetStageInputs( - FlutterGPUUnlitVertexShader::kAllShaderStageInputs, - FlutterGPUUnlitVertexShader::kInterleavedBufferLayout); - shaders["UnlitVertex"] = OpenRuntimeStageAsShader( - std::make_shared(kFlutterGPUUnlitVertIPLR, - kFlutterGPUUnlitVertIPLRLength), - vertex_desc); - shaders["UnlitFragment"] = OpenRuntimeStageAsShader( - std::make_shared(kFlutterGPUUnlitFragIPLR, - kFlutterGPUUnlitFragIPLRLength), - nullptr); - } - { - auto vertex_desc = std::make_shared(); - vertex_desc->SetStageInputs( - FlutterGPUTextureVertexShader::kAllShaderStageInputs, - FlutterGPUTextureVertexShader::kInterleavedBufferLayout); - shaders["TextureVertex"] = OpenRuntimeStageAsShader( - std::make_shared( - kFlutterGPUTextureVertIPLR, kFlutterGPUTextureVertIPLRLength), - vertex_desc); - shaders["TextureFragment"] = OpenRuntimeStageAsShader( - std::make_shared( - kFlutterGPUTextureFragIPLR, kFlutterGPUTextureFragIPLRLength), - nullptr); - } - auto library = ShaderLibrary::MakeFromShaders(std::move(shaders)); - return library; -} -// ===[ END MEMES ]============================================================= - IMPLEMENT_WRAPPERTYPEINFO(flutter_gpu, ShaderLibrary); fml::RefPtr ShaderLibrary::override_shader_library_; @@ -72,20 +30,21 @@ fml::RefPtr ShaderLibrary::override_shader_library_; fml::RefPtr ShaderLibrary::MakeFromAsset( const std::string& name, std::string& out_error) { - // =========================================================================== - // This is a temporary hack to get the shader library populated in the - // framework before the shader bundle format is landed! - if (!override_shader_library_) { - return InstantiateTestShaderLibrary(); - } - // =========================================================================== - if (override_shader_library_) { return override_shader_library_; } - // TODO(bdero): Load the ShaderLibrary asset. - out_error = "Shader bundle asset unimplemented"; - return nullptr; + + auto dart_state = UIDartState::Current(); + std::shared_ptr asset_manager = + dart_state->platform_configuration()->client()->GetAssetManager(); + + std::unique_ptr data = asset_manager->GetAsMapping(name); + if (data == nullptr) { + out_error = std::string("Asset '") + name + std::string("' not found."); + return nullptr; + } + + return MakeFromFlatbuffer(std::move(data)); } fml::RefPtr ShaderLibrary::MakeFromShaders(ShaderMap shaders) { From 0f126264c7e8728524c2bd9906408f2b9e21cdcc Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Sun, 10 Dec 2023 01:44:07 -0800 Subject: [PATCH 15/27] Asset dependency --- lib/gpu/BUILD.gn | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/gpu/BUILD.gn b/lib/gpu/BUILD.gn index da7da8eeae0b9..abaad37932437 100644 --- a/lib/gpu/BUILD.gn +++ b/lib/gpu/BUILD.gn @@ -61,6 +61,7 @@ source_set("gpu") { ] } deps = [ + "//flutter/assets", "//flutter/impeller", "//flutter/impeller/display_list:skia_conversions", "//flutter/impeller/shader_bundle:shader_bundle_flatbuffers", From 1da673ed6146d6104284feedeb9ffd2d2fea91de Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Sun, 10 Dec 2023 20:33:19 -0800 Subject: [PATCH 16/27] Hold on to dart-side refs for shader libraries --- lib/gpu/lib/src/shader_library.dart | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/gpu/lib/src/shader_library.dart b/lib/gpu/lib/src/shader_library.dart index 551c20bc589ee..636229d7042a5 100644 --- a/lib/gpu/lib/src/shader_library.dart +++ b/lib/gpu/lib/src/shader_library.dart @@ -18,6 +18,11 @@ base class ShaderLibrary extends NativeFieldWrapperClass1 { ShaderLibrary._(); + // Hold a Dart-side reference to shaders in the library as they're wrapped for + // the first time. This prevents the wrapper from getting prematurely + // destroyed. + Map shaders_ = {}; + Shader? operator [](String shaderName) { // This `flutter_gpu` library isn't always registered as part of the builtin // DartClassLibrary, and so we can't instantiate the Dart classes on the @@ -25,7 +30,14 @@ base class ShaderLibrary extends NativeFieldWrapperClass1 { // Providing a new wrapper to [_getShader] for wrapping the native // counterpart (if it hasn't been wrapped already) is a hack to work around // this. - return _getShader(shaderName, Shader._()); + Shader? result = shaders_[shaderName]; + if (result == null) { + result = _getShader(shaderName, Shader._()); + if (result != null) { + shaders_[shaderName] = result; + } + } + return result; } @Native( From 98f3cf5fabd6d8f3af0ea72e6bb48ac385d532d9 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Sun, 10 Dec 2023 20:41:37 -0800 Subject: [PATCH 17/27] Remove excess runtime classes that didn't end up getting used --- impeller/compiler/BUILD.gn | 2 +- impeller/shader_bundle/BUILD.gn | 16 ---------- impeller/shader_bundle/shader.cc | 21 ------------- impeller/shader_bundle/shader.h | 36 ---------------------- impeller/shader_bundle/shader_bundle.cc | 40 ------------------------- impeller/shader_bundle/shader_bundle.h | 35 ---------------------- 6 files changed, 1 insertion(+), 149 deletions(-) delete mode 100644 impeller/shader_bundle/shader.cc delete mode 100644 impeller/shader_bundle/shader.h delete mode 100644 impeller/shader_bundle/shader_bundle.cc delete mode 100644 impeller/shader_bundle/shader_bundle.h diff --git a/impeller/compiler/BUILD.gn b/impeller/compiler/BUILD.gn index d279984ea73fe..ac782a18d98e3 100644 --- a/impeller/compiler/BUILD.gn +++ b/impeller/compiler/BUILD.gn @@ -67,10 +67,10 @@ impeller_component("compiler_lib") { "../base", "../geometry", "../runtime_stage", - "../shader_bundle", "//flutter/fml", # All third_party deps must be included by the global license script. + "//flutter/impeller/shader_bundle:shader_bundle_flatbuffers", "//third_party/inja", "//third_party/shaderc_flutter", "//third_party/spirv_cross_flutter", diff --git a/impeller/shader_bundle/BUILD.gn b/impeller/shader_bundle/BUILD.gn index 556a56e480d2a..fd58931d755ef 100644 --- a/impeller/shader_bundle/BUILD.gn +++ b/impeller/shader_bundle/BUILD.gn @@ -14,19 +14,3 @@ flatbuffers("shader_bundle_flatbuffers") { "//flutter/third_party/flatbuffers", ] } - -impeller_component("shader_bundle") { - sources = [ - "shader.cc", - "shader.h", - "shader_bundle.cc", - "shader_bundle.h", - ] - public_deps = [ - ":shader_bundle_flatbuffers", - "../base", - "../core", - "//flutter/fml", - "//flutter/impeller/runtime_stage:runtime_stage", - ] -} diff --git a/impeller/shader_bundle/shader.cc b/impeller/shader_bundle/shader.cc deleted file mode 100644 index 67b47aefd2589..0000000000000 --- a/impeller/shader_bundle/shader.cc +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "impeller/shader_bundle/shader.h" - -namespace impeller { - -BundledShader::BundledShader() = default; - -BundledShader::BundledShader(const fb::Shader* shader) - : runtime_stage_( - std::make_shared(RuntimeStage(shader->shader()))) { - is_valid_ = runtime_stage_->IsValid(); -} - -bool BundledShader::IsValid() const { - return is_valid_; -} - -} // namespace impeller diff --git a/impeller/shader_bundle/shader.h b/impeller/shader_bundle/shader.h deleted file mode 100644 index d446012c882e6..0000000000000 --- a/impeller/shader_bundle/shader.h +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#pragma once - -#include -#include - -#include "flutter/fml/mapping.h" -#include "impeller/runtime_stage/runtime_stage.h" -#include "impeller/shader_bundle/shader_bundle_flatbuffers.h" - -namespace impeller { - -class ShaderBundle; - -class BundledShader { - public: - // Note: Default constructor and copy operations required for map usage in - // ShaderBundle. - BundledShader(); - - bool IsValid() const; - - private: - bool is_valid_ = false; - - std::shared_ptr runtime_stage_; - - explicit BundledShader(const fb::Shader* shader); - - friend ShaderBundle; -}; - -} // namespace impeller diff --git a/impeller/shader_bundle/shader_bundle.cc b/impeller/shader_bundle/shader_bundle.cc deleted file mode 100644 index 6a7faa4e7fe57..0000000000000 --- a/impeller/shader_bundle/shader_bundle.cc +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "impeller/shader_bundle/shader_bundle.h" - -#include "impeller/shader_bundle/shader_bundle_flatbuffers.h" - -namespace impeller { - -ShaderBundle::ShaderBundle(std::shared_ptr payload) - : payload_(std::move(payload)) { - if (payload_ == nullptr || !payload_->GetMapping()) { - return; - } - if (!fb::ShaderBundleBufferHasIdentifier(payload_->GetMapping())) { - return; - } - auto shader_bundle = fb::GetShaderBundle(payload_->GetMapping()); - if (!shader_bundle) { - return; - } - - auto* shaders = shader_bundle->shaders(); - for (size_t i = 0; i < shaders->size(); i++) { - const fb::Shader* shader = shaders->Get(i); - shaders_[shader->name()->str()] = BundledShader(shader); - if (!shaders_[shader->name()->str()].IsValid()) { - return; - } - } - - is_valid_ = true; -} - -bool ShaderBundle::IsValid() const { - return is_valid_; -} - -} // namespace impeller diff --git a/impeller/shader_bundle/shader_bundle.h b/impeller/shader_bundle/shader_bundle.h deleted file mode 100644 index 6decefab2fd36..0000000000000 --- a/impeller/shader_bundle/shader_bundle.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#pragma once - -#include -#include - -#include "flutter/fml/mapping.h" -#include "impeller/runtime_stage/runtime_stage.h" -#include "impeller/shader_bundle/shader.h" - -namespace impeller { - -class ShaderBundle { - public: - explicit ShaderBundle(std::shared_ptr payload); - - ~ShaderBundle(); - ShaderBundle(ShaderBundle&&); - - bool IsValid() const; - - private: - bool is_valid_; - - std::shared_ptr payload_; - std::map shaders_; - - ShaderBundle(const ShaderBundle&) = delete; - ShaderBundle& operator=(const ShaderBundle&) = delete; -}; - -} // namespace impeller From ce7ee8dbd7e2953572a99a2e9e68be7cf203a5db Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Sun, 10 Dec 2023 22:44:55 -0800 Subject: [PATCH 18/27] Licenses --- ci/licenses_golden/licenses_flutter | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 99c8122d0d994..5c379ed01c12e 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -4933,6 +4933,8 @@ ORIGIN: ../../../flutter/impeller/compiler/reflector.cc + ../../../flutter/LICEN ORIGIN: ../../../flutter/impeller/compiler/reflector.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/compiler/runtime_stage_data.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/compiler/runtime_stage_data.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/compiler/shader_bundle.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/compiler/shader_bundle.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/compiler/shader_lib/flutter/runtime_effect.glsl + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/compiler/shader_lib/impeller/blending.glsl + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/compiler/shader_lib/impeller/branching.glsl + ../../../flutter/LICENSE @@ -5473,6 +5475,7 @@ ORIGIN: ../../../flutter/impeller/runtime_stage/runtime_stage.fbs + ../../../flu ORIGIN: ../../../flutter/impeller/runtime_stage/runtime_stage.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/runtime_stage/runtime_stage_playground.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/runtime_stage/runtime_stage_playground.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/runtime_stage/runtime_stage_types.fbs + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/scene/animation/animation.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/scene/animation/animation.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/scene/animation/animation_clip.cc + ../../../flutter/LICENSE @@ -5527,6 +5530,7 @@ ORIGIN: ../../../flutter/impeller/shader_archive/shader_archive_main.cc + ../../ ORIGIN: ../../../flutter/impeller/shader_archive/shader_archive_types.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/shader_archive/shader_archive_writer.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/shader_archive/shader_archive_writer.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/shader_bundle/shader_bundle.fbs + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/tessellator/c/tessellator.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/tessellator/c/tessellator.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/tessellator/dart/lib/tessellator.dart + ../../../flutter/LICENSE @@ -7734,6 +7738,8 @@ FILE: ../../../flutter/impeller/compiler/reflector.cc FILE: ../../../flutter/impeller/compiler/reflector.h FILE: ../../../flutter/impeller/compiler/runtime_stage_data.cc FILE: ../../../flutter/impeller/compiler/runtime_stage_data.h +FILE: ../../../flutter/impeller/compiler/shader_bundle.cc +FILE: ../../../flutter/impeller/compiler/shader_bundle.h FILE: ../../../flutter/impeller/compiler/shader_lib/flutter/runtime_effect.glsl FILE: ../../../flutter/impeller/compiler/shader_lib/impeller/blending.glsl FILE: ../../../flutter/impeller/compiler/shader_lib/impeller/branching.glsl @@ -8275,6 +8281,7 @@ FILE: ../../../flutter/impeller/runtime_stage/runtime_stage.fbs FILE: ../../../flutter/impeller/runtime_stage/runtime_stage.h FILE: ../../../flutter/impeller/runtime_stage/runtime_stage_playground.cc FILE: ../../../flutter/impeller/runtime_stage/runtime_stage_playground.h +FILE: ../../../flutter/impeller/runtime_stage/runtime_stage_types.fbs FILE: ../../../flutter/impeller/scene/animation/animation.cc FILE: ../../../flutter/impeller/scene/animation/animation.h FILE: ../../../flutter/impeller/scene/animation/animation_clip.cc @@ -8329,6 +8336,7 @@ FILE: ../../../flutter/impeller/shader_archive/shader_archive_main.cc FILE: ../../../flutter/impeller/shader_archive/shader_archive_types.h FILE: ../../../flutter/impeller/shader_archive/shader_archive_writer.cc FILE: ../../../flutter/impeller/shader_archive/shader_archive_writer.h +FILE: ../../../flutter/impeller/shader_bundle/shader_bundle.fbs FILE: ../../../flutter/impeller/tessellator/c/tessellator.cc FILE: ../../../flutter/impeller/tessellator/c/tessellator.h FILE: ../../../flutter/impeller/tessellator/dart/lib/tessellator.dart From e57b927a7394f41fba5337a1bbda2a1ec07ab28e Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Mon, 11 Dec 2023 14:50:02 -0800 Subject: [PATCH 19/27] Start of tests --- impeller/compiler/BUILD.gn | 1 + impeller/compiler/shader_bundle.cc | 4 ++-- impeller/compiler/shader_bundle.h | 3 +++ impeller/compiler/shader_bundle_unittests.cc | 24 ++++++++++++++++++++ impeller/compiler/switches_unittests.cc | 13 +++++++++++ 5 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 impeller/compiler/shader_bundle_unittests.cc diff --git a/impeller/compiler/BUILD.gn b/impeller/compiler/BUILD.gn index ac782a18d98e3..1debfcbf71d54 100644 --- a/impeller/compiler/BUILD.gn +++ b/impeller/compiler/BUILD.gn @@ -105,6 +105,7 @@ impeller_component("compiler_unittests") { "compiler_test.cc", "compiler_test.h", "compiler_unittests.cc", + "shader_bundle_unittests.cc", "switches_unittests.cc", ] diff --git a/impeller/compiler/shader_bundle.cc b/impeller/compiler/shader_bundle.cc index 425c361ae5661..f181f2a3c86af 100644 --- a/impeller/compiler/shader_bundle.cc +++ b/impeller/compiler/shader_bundle.cc @@ -15,8 +15,8 @@ namespace impeller { namespace compiler { -static std::optional ParseShaderBundleConfig( - std::string& json_config) { +std::optional ParseShaderBundleConfig( + const std::string& json_config) { auto json = nlohmann::json::parse(json_config); if (!json.is_object()) { std::cerr << "The shader bundle must be a JSON object." << std::endl; diff --git a/impeller/compiler/shader_bundle.h b/impeller/compiler/shader_bundle.h index 44be41f4ec29a..77a4e038cd234 100644 --- a/impeller/compiler/shader_bundle.h +++ b/impeller/compiler/shader_bundle.h @@ -8,6 +8,9 @@ namespace impeller { namespace compiler { +std::optional ParseShaderBundleConfig( + const std::string& json_config); + bool GenerateShaderBundle(Switches& switches, SourceOptions& options); } // namespace compiler diff --git a/impeller/compiler/shader_bundle_unittests.cc b/impeller/compiler/shader_bundle_unittests.cc new file mode 100644 index 0000000000000..efaf529918471 --- /dev/null +++ b/impeller/compiler/shader_bundle_unittests.cc @@ -0,0 +1,24 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include +#include + +#include "impeller/compiler/shader_bundle.h" + +#include "flutter/testing/testing.h" + +namespace impeller { +namespace compiler { +namespace testing { + +TEST(ShaderBundleTest, ShaderBundleConfigFailsForInvalidJSON) { + std::string bundle = ""; + auto result = ParseShaderBundleConfig(bundle); + ASSERT_FALSE(result.has_value()); +} + +} // namespace testing +} // namespace compiler +} // namespace impeller diff --git a/impeller/compiler/switches_unittests.cc b/impeller/compiler/switches_unittests.cc index 92b03ecc2a1fe..fd1591e193f0d 100644 --- a/impeller/compiler/switches_unittests.cc +++ b/impeller/compiler/switches_unittests.cc @@ -80,6 +80,19 @@ TEST(SwitchesTEst, ConvertToEntrypointName) { ASSERT_EQ(ConvertToEntrypointName(""), ""); } +TEST(SwitchesTest, ShaderBundleModeValid) { + // Shader bundles process multiple shaders, and so the single-file input/spirv + // flags are not required. + std::vector options = { + "--shader-bundle={}", "--sl=test.shaderbundle", "--runtime-stage-metal"}; + + auto cl = fml::CommandLineFromIteratorsWithArgv0("impellerc", options.begin(), + options.end()); + Switches switches(cl); + ASSERT_TRUE(switches.AreValid(std::cout)); + ASSERT_EQ(switches.shader_bundle, "{}"); +} + } // namespace testing } // namespace compiler } // namespace impeller From c9176dc9084e2dd4fd96912c451574f8c49af763 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Mon, 11 Dec 2023 15:19:07 -0800 Subject: [PATCH 20/27] Error message tests --- impeller/compiler/shader_bundle.cc | 34 +++++++++++--------- impeller/compiler/shader_bundle.h | 3 +- impeller/compiler/shader_bundle_unittests.cc | 30 ++++++++++++++++- 3 files changed, 49 insertions(+), 18 deletions(-) diff --git a/impeller/compiler/shader_bundle.cc b/impeller/compiler/shader_bundle.cc index f181f2a3c86af..9835a7e14151a 100644 --- a/impeller/compiler/shader_bundle.cc +++ b/impeller/compiler/shader_bundle.cc @@ -16,44 +16,46 @@ namespace impeller { namespace compiler { std::optional ParseShaderBundleConfig( - const std::string& json_config) { + const std::string& json_config, + std::ostream& error_stream) { auto json = nlohmann::json::parse(json_config); if (!json.is_object()) { - std::cerr << "The shader bundle must be a JSON object." << std::endl; + error_stream << "The shader bundle must be a JSON object." << std::endl; return std::nullopt; } ShaderBundleConfig bundle; for (auto& [shader_name, shader_value] : json.items()) { if (bundle.find(shader_name) != bundle.end()) { - std::cerr << "Duplicate shader \"" << shader_name << "\"." << std::endl; + error_stream << "Duplicate shader \"" << shader_name << "\"." + << std::endl; return std::nullopt; } if (!shader_value.is_object()) { - std::cerr << "Invalid shader entry \"" << shader_name - << "\": Entry is not a JSON object." << std::endl; + error_stream << "Invalid shader entry \"" << shader_name + << "\": Entry is not a JSON object." << std::endl; return std::nullopt; } ShaderConfig shader; if (!shader_value.contains("file")) { - std::cerr << "Invalid shader entry \"" << shader_name - << "\": Missing required \"file\" field. \"" << std::endl; + error_stream << "Invalid shader entry \"" << shader_name + << "\": Missing required \"file\" field. \"" << std::endl; return std::nullopt; } shader.source_file_name = shader_value["file"]; if (!shader_value.contains("type")) { - std::cerr << "Invalid shader entry \"" << shader_name - << "\": Missing required \"type\" field. \"" << std::endl; + error_stream << "Invalid shader entry \"" << shader_name + << "\": Missing required \"type\" field. \"" << std::endl; return std::nullopt; } shader.type = SourceTypeFromString(shader_value["type"]); if (shader.type == SourceType::kUnknown) { - std::cerr << "Invalid shader entry \"" << shader_name - << "\": Shader type \"" << shader_value["type"] - << "\" is unknown. \"" << std::endl; + error_stream << "Invalid shader entry \"" << shader_name + << "\": Shader type \"" << shader_value["type"] + << "\" is unknown. \"" << std::endl; return std::nullopt; } @@ -61,9 +63,9 @@ std::optional ParseShaderBundleConfig( ? ToSourceLanguage(shader_value["language"]) : SourceLanguage::kGLSL; if (shader.language == SourceLanguage::kUnknown) { - std::cerr << "Invalid shader entry \"" << shader_name - << "\": Unknown language type \"" << shader_value["language"] - << "\"." << std::endl; + error_stream << "Invalid shader entry \"" << shader_name + << "\": Unknown language type \"" << shader_value["language"] + << "\"." << std::endl; return std::nullopt; } @@ -146,7 +148,7 @@ bool GenerateShaderBundle(Switches& switches, SourceOptions& options) { /// std::optional bundle_config = - ParseShaderBundleConfig(switches.shader_bundle); + ParseShaderBundleConfig(switches.shader_bundle, std::cerr); if (!bundle_config) { return false; } diff --git a/impeller/compiler/shader_bundle.h b/impeller/compiler/shader_bundle.h index 77a4e038cd234..5c97f2a6f21ba 100644 --- a/impeller/compiler/shader_bundle.h +++ b/impeller/compiler/shader_bundle.h @@ -9,7 +9,8 @@ namespace impeller { namespace compiler { std::optional ParseShaderBundleConfig( - const std::string& json_config); + const std::string& json_config, + std::ostream& error_stream); bool GenerateShaderBundle(Switches& switches, SourceOptions& options); diff --git a/impeller/compiler/shader_bundle_unittests.cc b/impeller/compiler/shader_bundle_unittests.cc index efaf529918471..c23738eab0547 100644 --- a/impeller/compiler/shader_bundle_unittests.cc +++ b/impeller/compiler/shader_bundle_unittests.cc @@ -13,10 +13,38 @@ namespace impeller { namespace compiler { namespace testing { +const std::string kUnlitFragmentBundleConfig = + "\"UnlitFragment\": {\"type\": \"fragment\", \"file\": " + "\"shaders/flutter_gpu_unlit.frag\"}"; +const std::string kUnlitVertexBundleConfig = + "\"UnlitVertex\": {\"type\": \"vertex\", \"file\": " + "\"shaders/flutter_gpu_unlit.vert\"}"; + TEST(ShaderBundleTest, ShaderBundleConfigFailsForInvalidJSON) { std::string bundle = ""; - auto result = ParseShaderBundleConfig(bundle); + std::stringstream error; + auto result = ParseShaderBundleConfig(bundle, error); + ASSERT_FALSE(result.has_value()); + ASSERT_STREQ(error.str().c_str(), "The shader bundle must be a JSON object."); +} + +TEST(ShaderBundleTest, ShaderBundleConfigFailsForDuplicateShaders) { + std::string bundle = "{" + kUnlitFragmentBundleConfig + ", " + + kUnlitFragmentBundleConfig + "}"; + std::stringstream error; + auto result = ParseShaderBundleConfig(bundle, error); + ASSERT_FALSE(result.has_value()); + ASSERT_STREQ(error.str().c_str(), "Duplicate shader \"UnlitFragment\"."); +} + +TEST(ShaderBundleTest, ShaderBundleConfigFailsWhenEntryNotObject) { + std::string bundle = "\"UnlitVertex\": []"; + std::stringstream error; + auto result = ParseShaderBundleConfig(bundle, error); ASSERT_FALSE(result.has_value()); + ASSERT_STREQ( + error.str().c_str(), + "Invalid shader entry \"UnlitFragment\": Entry is not a JSON object."); } } // namespace testing From cd5ba507f39e1691cedaa59774621384a07adf44 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Mon, 11 Dec 2023 18:56:24 -0800 Subject: [PATCH 21/27] Remainder of parser tests --- ci/licenses_golden/licenses_flutter | 2 + impeller/compiler/shader_bundle.cc | 19 ++-- impeller/compiler/shader_bundle_unittests.cc | 93 +++++++++++++++++--- 3 files changed, 91 insertions(+), 23 deletions(-) diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 5c379ed01c12e..661219ad84a55 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -4933,6 +4933,7 @@ ORIGIN: ../../../flutter/impeller/compiler/reflector.cc + ../../../flutter/LICEN ORIGIN: ../../../flutter/impeller/compiler/reflector.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/compiler/runtime_stage_data.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/compiler/runtime_stage_data.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/compiler/shader_bundle_unittests.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/compiler/shader_bundle.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/compiler/shader_bundle.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/compiler/shader_lib/flutter/runtime_effect.glsl + ../../../flutter/LICENSE @@ -7738,6 +7739,7 @@ FILE: ../../../flutter/impeller/compiler/reflector.cc FILE: ../../../flutter/impeller/compiler/reflector.h FILE: ../../../flutter/impeller/compiler/runtime_stage_data.cc FILE: ../../../flutter/impeller/compiler/runtime_stage_data.h +FILE: ../../../flutter/impeller/compiler/shader_bundle_unittests.cc FILE: ../../../flutter/impeller/compiler/shader_bundle.cc FILE: ../../../flutter/impeller/compiler/shader_bundle.h FILE: ../../../flutter/impeller/compiler/shader_lib/flutter/runtime_effect.glsl diff --git a/impeller/compiler/shader_bundle.cc b/impeller/compiler/shader_bundle.cc index 9835a7e14151a..e0edb51ea49a0 100644 --- a/impeller/compiler/shader_bundle.cc +++ b/impeller/compiler/shader_bundle.cc @@ -18,9 +18,10 @@ namespace compiler { std::optional ParseShaderBundleConfig( const std::string& json_config, std::ostream& error_stream) { - auto json = nlohmann::json::parse(json_config); - if (!json.is_object()) { - error_stream << "The shader bundle must be a JSON object." << std::endl; + auto json = nlohmann::json::parse(json_config, nullptr, false); + if (json.is_discarded() || !json.is_object()) { + error_stream << "The shader bundle is not a valid JSON object." + << std::endl; return std::nullopt; } @@ -41,21 +42,21 @@ std::optional ParseShaderBundleConfig( if (!shader_value.contains("file")) { error_stream << "Invalid shader entry \"" << shader_name - << "\": Missing required \"file\" field. \"" << std::endl; + << "\": Missing required \"file\" field." << std::endl; return std::nullopt; } shader.source_file_name = shader_value["file"]; if (!shader_value.contains("type")) { error_stream << "Invalid shader entry \"" << shader_name - << "\": Missing required \"type\" field. \"" << std::endl; + << "\": Missing required \"type\" field." << std::endl; return std::nullopt; } shader.type = SourceTypeFromString(shader_value["type"]); if (shader.type == SourceType::kUnknown) { error_stream << "Invalid shader entry \"" << shader_name - << "\": Shader type \"" << shader_value["type"] - << "\" is unknown. \"" << std::endl; + << "\": Shader type " << shader_value["type"] + << " is unknown." << std::endl; return std::nullopt; } @@ -64,8 +65,8 @@ std::optional ParseShaderBundleConfig( : SourceLanguage::kGLSL; if (shader.language == SourceLanguage::kUnknown) { error_stream << "Invalid shader entry \"" << shader_name - << "\": Unknown language type \"" << shader_value["language"] - << "\"." << std::endl; + << "\": Unknown language type " << shader_value["language"] + << "." << std::endl; return std::nullopt; } diff --git a/impeller/compiler/shader_bundle_unittests.cc b/impeller/compiler/shader_bundle_unittests.cc index c23738eab0547..e3520a271f71c 100644 --- a/impeller/compiler/shader_bundle_unittests.cc +++ b/impeller/compiler/shader_bundle_unittests.cc @@ -2,12 +2,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include -#include - #include "impeller/compiler/shader_bundle.h" #include "flutter/testing/testing.h" +#include "impeller/compiler/types.h" namespace impeller { namespace compiler { @@ -20,31 +18,98 @@ const std::string kUnlitVertexBundleConfig = "\"UnlitVertex\": {\"type\": \"vertex\", \"file\": " "\"shaders/flutter_gpu_unlit.vert\"}"; -TEST(ShaderBundleTest, ShaderBundleConfigFailsForInvalidJSON) { +TEST(ShaderBundleTest, ParseShaderBundleConfigFailsForInvalidJSON) { std::string bundle = ""; std::stringstream error; auto result = ParseShaderBundleConfig(bundle, error); ASSERT_FALSE(result.has_value()); - ASSERT_STREQ(error.str().c_str(), "The shader bundle must be a JSON object."); + ASSERT_STREQ(error.str().c_str(), + "The shader bundle is not a valid JSON object.\n"); } -TEST(ShaderBundleTest, ShaderBundleConfigFailsForDuplicateShaders) { - std::string bundle = "{" + kUnlitFragmentBundleConfig + ", " + - kUnlitFragmentBundleConfig + "}"; +TEST(ShaderBundleTest, ParseShaderBundleConfigFailsWhenEntryNotObject) { + std::string bundle = "{\"UnlitVertex\": []}"; std::stringstream error; auto result = ParseShaderBundleConfig(bundle, error); ASSERT_FALSE(result.has_value()); - ASSERT_STREQ(error.str().c_str(), "Duplicate shader \"UnlitFragment\"."); + ASSERT_STREQ( + error.str().c_str(), + "Invalid shader entry \"UnlitVertex\": Entry is not a JSON object.\n"); } -TEST(ShaderBundleTest, ShaderBundleConfigFailsWhenEntryNotObject) { - std::string bundle = "\"UnlitVertex\": []"; +TEST(ShaderBundleTest, ParseShaderBundleConfigFailsWhenMissingFile) { + std::string bundle = "{\"UnlitVertex\": {\"type\": \"vertex\"}}"; std::stringstream error; auto result = ParseShaderBundleConfig(bundle, error); ASSERT_FALSE(result.has_value()); - ASSERT_STREQ( - error.str().c_str(), - "Invalid shader entry \"UnlitFragment\": Entry is not a JSON object."); + ASSERT_STREQ(error.str().c_str(), + "Invalid shader entry \"UnlitVertex\": Missing required " + "\"file\" field.\n"); +} + +TEST(ShaderBundleTest, ParseShaderBundleConfigFailsWhenMissingType) { + std::string bundle = + "{\"UnlitVertex\": {\"file\": \"shaders/flutter_gpu_unlit.vert\"}}"; + std::stringstream error; + auto result = ParseShaderBundleConfig(bundle, error); + ASSERT_FALSE(result.has_value()); + ASSERT_STREQ(error.str().c_str(), + "Invalid shader entry \"UnlitVertex\": Missing required " + "\"type\" field.\n"); +} + +TEST(ShaderBundleTest, ParseShaderBundleConfigFailsForInvalidType) { + std::string bundle = + "{\"UnlitVertex\": {\"type\": \"invalid\", \"file\": " + "\"shaders/flutter_gpu_unlit.vert\"}}"; + std::stringstream error; + auto result = ParseShaderBundleConfig(bundle, error); + ASSERT_FALSE(result.has_value()); + ASSERT_STREQ(error.str().c_str(), + "Invalid shader entry \"UnlitVertex\": Shader type " + "\"invalid\" is unknown.\n"); +} + +TEST(ShaderBundleTest, ParseShaderBundleConfigFailsForInvalidLanguage) { + std::string bundle = + "{\"UnlitVertex\": {\"type\": \"vertex\", \"language\": \"invalid\", " + "\"file\": \"shaders/flutter_gpu_unlit.vert\"}}"; + std::stringstream error; + auto result = ParseShaderBundleConfig(bundle, error); + ASSERT_FALSE(result.has_value()); + ASSERT_STREQ(error.str().c_str(), + "Invalid shader entry \"UnlitVertex\": Unknown language type " + "\"invalid\".\n"); +} + +TEST(ShaderBundleTest, ParseShaderBundleConfigReturnsExpectedConfig) { + std::string bundle = + "{" + kUnlitVertexBundleConfig + ", " + kUnlitFragmentBundleConfig + "}"; + std::stringstream error; + auto result = ParseShaderBundleConfig(bundle, error); + ASSERT_TRUE(result.has_value()); + ASSERT_STREQ(error.str().c_str(), ""); + + // NOLINTBEGIN(bugprone-unchecked-optional-access) + auto maybe_vertex = result->find("UnlitVertex"); + auto maybe_fragment = result->find("UnlitFragment"); + ASSERT_TRUE(maybe_vertex != result->end()); + ASSERT_TRUE(maybe_fragment != result->end()); + auto vertex = maybe_vertex->second; + auto fragment = maybe_fragment->second; + // NOLINTEND(bugprone-unchecked-optional-access) + + EXPECT_EQ(vertex.type, SourceType::kVertexShader); + EXPECT_EQ(vertex.language, SourceLanguage::kGLSL); + EXPECT_STREQ(vertex.entry_point.c_str(), "main"); + EXPECT_STREQ(vertex.source_file_name.c_str(), + "shaders/flutter_gpu_unlit.vert"); + + EXPECT_EQ(fragment.type, SourceType::kFragmentShader); + EXPECT_EQ(fragment.language, SourceLanguage::kGLSL); + EXPECT_STREQ(fragment.entry_point.c_str(), "main"); + EXPECT_STREQ(fragment.source_file_name.c_str(), + "shaders/flutter_gpu_unlit.frag"); } } // namespace testing From ce28f95adb403a97968fd909f9dfd9d97727e78d Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Mon, 11 Dec 2023 23:37:18 -0800 Subject: [PATCH 22/27] Full bundle->flatbuffer test --- impeller/compiler/shader_bundle.cc | 39 +++++--- impeller/compiler/shader_bundle.h | 19 +++- impeller/compiler/shader_bundle_unittests.cc | 97 ++++++++++++++++++++ impeller/fixtures/BUILD.gn | 4 + 4 files changed, 147 insertions(+), 12 deletions(-) diff --git a/impeller/compiler/shader_bundle.cc b/impeller/compiler/shader_bundle.cc index e0edb51ea49a0..369d1bc588b8a 100644 --- a/impeller/compiler/shader_bundle.cc +++ b/impeller/compiler/shader_bundle.cc @@ -16,9 +16,9 @@ namespace impeller { namespace compiler { std::optional ParseShaderBundleConfig( - const std::string& json_config, + const std::string& bundle_config_json, std::ostream& error_stream) { - auto json = nlohmann::json::parse(json_config, nullptr, false); + auto json = nlohmann::json::parse(bundle_config_json, nullptr, false); if (json.is_discarded() || !json.is_object()) { error_stream << "The shader bundle is not a valid JSON object." << std::endl; @@ -141,17 +141,17 @@ static std::unique_ptr GenerateShaderFB( return result; } -/// Parses the given JSON shader bundle configuration and invokes the compiler -/// multiple times to produce a shader bundle file. -bool GenerateShaderBundle(Switches& switches, SourceOptions& options) { +std::optional GenerateShaderBundleFlatbuffer( + const std::string& bundle_config_json, + SourceOptions& options) { // -------------------------------------------------------------------------- /// 1. Parse the bundle configuration. /// std::optional bundle_config = - ParseShaderBundleConfig(switches.shader_bundle, std::cerr); + ParseShaderBundleConfig(bundle_config_json, std::cerr); if (!bundle_config) { - return false; + return std::nullopt; } // -------------------------------------------------------------------------- @@ -164,18 +164,35 @@ bool GenerateShaderBundle(Switches& switches, SourceOptions& options) { std::unique_ptr shader = GenerateShaderFB(options, shader_name, shader_config); if (!shader) { - return false; + return std::nullopt; } shader_bundle.shaders.push_back(std::move(shader)); } + return shader_bundle; +} + +bool GenerateShaderBundle(Switches& switches, SourceOptions& options) { + // -------------------------------------------------------------------------- + /// 1. Parse the shader bundle and generate the flatbuffer result. + /// + + auto shader_bundle = + GenerateShaderBundleFlatbuffer(switches.shader_bundle, options); + if (!shader_bundle.has_value()) { + // Specific error messages are already handled by + // GenerateShaderBundleFlatbuffer. + return false; + } + // -------------------------------------------------------------------------- - /// 3. Serialize the shader bundle and write to disk. + /// 2. Serialize the shader bundle and write to disk. /// auto builder = std::make_shared(); - builder->Finish(fb::ShaderBundle::Pack(*builder.get(), &shader_bundle), - fb::ShaderBundleIdentifier()); + builder->Finish( + fb::ShaderBundle::Pack(*builder.get(), &shader_bundle.value()), + fb::ShaderBundleIdentifier()); auto mapping = std::make_shared( builder->GetBufferPointer(), builder->GetSize(), [builder](auto, auto) {}); diff --git a/impeller/compiler/shader_bundle.h b/impeller/compiler/shader_bundle.h index 5c97f2a6f21ba..438e5a56d754f 100644 --- a/impeller/compiler/shader_bundle.h +++ b/impeller/compiler/shader_bundle.h @@ -4,14 +4,31 @@ #include "impeller/compiler/source_options.h" #include "impeller/compiler/switches.h" +#include "impeller/shader_bundle/shader_bundle_flatbuffers.h" namespace impeller { namespace compiler { +/// @brief Parse a shader bundle configuration from a given JSON string. +/// +/// @note Exposed only for testing purposes. Use `GenerateShaderBundle` +/// directly. std::optional ParseShaderBundleConfig( - const std::string& json_config, + const std::string& bundle_config_json, std::ostream& error_stream); +/// @brief Parses the JSON shader bundle configuration and invokes the +/// compiler multiple times to produce a shader bundle flatbuffer. +/// +/// @note Exposed only for testing purposes. Use `GenerateShaderBundle` +/// directly. +std::optional GenerateShaderBundleFlatbuffer( + const std::string& bundle_config_json, + SourceOptions& options); + +/// @brief Parses the JSON shader bundle configuration and invokes the +/// compiler multiple times to produce a shader bundle flatbuffer, which +/// is then output to the `sl` file. bool GenerateShaderBundle(Switches& switches, SourceOptions& options); } // namespace compiler diff --git a/impeller/compiler/shader_bundle_unittests.cc b/impeller/compiler/shader_bundle_unittests.cc index e3520a271f71c..375f366bf64a4 100644 --- a/impeller/compiler/shader_bundle_unittests.cc +++ b/impeller/compiler/shader_bundle_unittests.cc @@ -2,10 +2,14 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +#include "gtest/gtest.h" #include "impeller/compiler/shader_bundle.h" #include "flutter/testing/testing.h" +#include "impeller/compiler/source_options.h" #include "impeller/compiler/types.h" +#include "impeller/runtime_stage/runtime_stage_types_flatbuffers.h" +#include "impeller/shader_bundle/shader_bundle_flatbuffers.h" namespace impeller { namespace compiler { @@ -112,6 +116,99 @@ TEST(ShaderBundleTest, ParseShaderBundleConfigReturnsExpectedConfig) { "shaders/flutter_gpu_unlit.frag"); } +template +const T* FindByName(const std::vector>& collection, + const std::string& name) { + const auto maybe = std::find_if( + collection.begin(), collection.end(), + [&name](const std::unique_ptr& value) { return value->name == name; }); + if (maybe == collection.end()) { + return nullptr; + } + return maybe->get(); +} + +TEST(ShaderBundleTest, GenerateShaderBundleFlatbufferProducesCorrectResult) { + std::string fixtures_path = flutter::testing::GetFixturesPath(); + std::string config = + "{\"UnlitFragment\": {\"type\": \"fragment\", \"file\": \"" + + fixtures_path + + "/flutter_gpu_unlit.frag\"}, \"UnlitVertex\": {\"type\": " + "\"vertex\", \"file\": \"" + + fixtures_path + "/flutter_gpu_unlit.vert\"}}"; + + SourceOptions options; + options.target_platform = TargetPlatform::kRuntimeStageMetal; + options.source_language = SourceLanguage::kGLSL; + + std::optional bundle = + GenerateShaderBundleFlatbuffer(config, options); + ASSERT_TRUE(bundle.has_value()); + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + const auto& shaders = bundle->shaders; + const auto* vertex = FindByName(shaders, "UnlitVertex"); + const auto* fragment = FindByName(shaders, "UnlitFragment"); + ASSERT_NE(vertex, nullptr); + ASSERT_NE(fragment, nullptr); + + // -------------------------------------------------------------------------- + /// Verify vertex shader. + /// + + EXPECT_STREQ(vertex->shader->entrypoint.c_str(), + "flutter_gpu_unlit_vertex_main"); + EXPECT_EQ(vertex->shader->stage, fb::Stage::kVertex); + EXPECT_EQ(vertex->shader->target_platform, fb::TargetPlatform::kMetal); + + // Inputs. + ASSERT_EQ(vertex->shader->inputs.size(), 1u); + const auto& v_in_position = vertex->shader->inputs[0]; + EXPECT_STREQ(v_in_position->name.c_str(), "position"); + EXPECT_EQ(v_in_position->location, 0u); + EXPECT_EQ(v_in_position->set, 0u); + EXPECT_EQ(v_in_position->binding, 0u); + EXPECT_EQ(v_in_position->type, fb::InputDataType::kFloat); + EXPECT_EQ(v_in_position->bit_width, 32u); + EXPECT_EQ(v_in_position->vec_size, 2u); + EXPECT_EQ(v_in_position->columns, 1u); + EXPECT_EQ(v_in_position->offset, 0u); + + // Uniforms. + ASSERT_EQ(vertex->shader->uniforms.size(), 2u); + const auto* v_mvp = FindByName(vertex->shader->uniforms, "mvp"); + ASSERT_NE(v_mvp, nullptr); + EXPECT_EQ(v_mvp->location, 0u); + EXPECT_EQ(v_mvp->type, fb::UniformDataType::kFloat); + EXPECT_EQ(v_mvp->bit_width, 32u); + EXPECT_EQ(v_mvp->rows, 4u); + EXPECT_EQ(v_mvp->columns, 4u); + EXPECT_EQ(v_mvp->array_elements, 0u); + const auto* v_color = FindByName(vertex->shader->uniforms, "color"); + ASSERT_NE(v_color, nullptr); + EXPECT_EQ(v_color->location, 1u); + EXPECT_EQ(v_color->type, fb::UniformDataType::kFloat); + EXPECT_EQ(v_color->bit_width, 32u); + EXPECT_EQ(v_color->rows, 4u); + EXPECT_EQ(v_color->columns, 1u); + EXPECT_EQ(v_color->array_elements, 0u); + + // -------------------------------------------------------------------------- + /// Verify fragment shader. + /// + + EXPECT_STREQ(fragment->shader->entrypoint.c_str(), + "flutter_gpu_unlit_fragment_main"); + EXPECT_EQ(fragment->shader->stage, fb::Stage::kFragment); + EXPECT_EQ(fragment->shader->target_platform, fb::TargetPlatform::kMetal); + + // Inputs (not recorded for fragment shaders). + ASSERT_EQ(fragment->shader->inputs.size(), 0u); + + // Uniforms. + ASSERT_EQ(fragment->shader->inputs.size(), 0u); +} + } // namespace testing } // namespace compiler } // namespace impeller diff --git a/impeller/fixtures/BUILD.gn b/impeller/fixtures/BUILD.gn index 4c6fbcb046f2f..19d003344fc36 100644 --- a/impeller/fixtures/BUILD.gn +++ b/impeller/fixtures/BUILD.gn @@ -76,6 +76,10 @@ test_fixtures("file_fixtures") { "blue_noise.png", "boston.jpg", "embarcadero.jpg", + "flutter_gpu_unlit.frag", + "flutter_gpu_unlit.vert", + "flutter_gpu_texture.frag", + "flutter_gpu_texture.vert", "flutter_logo_baked.glb", "kalimba.jpg", "multiple_stages.hlsl", From 2146d31594df7a9b984284dbe214582a79c7a99e Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Mon, 11 Dec 2023 23:38:28 -0800 Subject: [PATCH 23/27] Licenses --- ci/licenses_golden/excluded_files | 1 + ci/licenses_golden/licenses_flutter | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/licenses_golden/excluded_files b/ci/licenses_golden/excluded_files index d525a6154aae0..db8bcd971146e 100644 --- a/ci/licenses_golden/excluded_files +++ b/ci/licenses_golden/excluded_files @@ -133,6 +133,7 @@ ../../../flutter/impeller/base/base_unittests.cc ../../../flutter/impeller/compiler/README.md ../../../flutter/impeller/compiler/compiler_unittests.cc +../../../flutter/impeller/compiler/shader_bundle_unittests.cc ../../../flutter/impeller/compiler/switches_unittests.cc ../../../flutter/impeller/core/allocator_unittests.cc ../../../flutter/impeller/display_list/dl_unittests.cc diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 661219ad84a55..5c379ed01c12e 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -4933,7 +4933,6 @@ ORIGIN: ../../../flutter/impeller/compiler/reflector.cc + ../../../flutter/LICEN ORIGIN: ../../../flutter/impeller/compiler/reflector.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/compiler/runtime_stage_data.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/compiler/runtime_stage_data.h + ../../../flutter/LICENSE -ORIGIN: ../../../flutter/impeller/compiler/shader_bundle_unittests.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/compiler/shader_bundle.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/compiler/shader_bundle.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/compiler/shader_lib/flutter/runtime_effect.glsl + ../../../flutter/LICENSE @@ -7739,7 +7738,6 @@ FILE: ../../../flutter/impeller/compiler/reflector.cc FILE: ../../../flutter/impeller/compiler/reflector.h FILE: ../../../flutter/impeller/compiler/runtime_stage_data.cc FILE: ../../../flutter/impeller/compiler/runtime_stage_data.h -FILE: ../../../flutter/impeller/compiler/shader_bundle_unittests.cc FILE: ../../../flutter/impeller/compiler/shader_bundle.cc FILE: ../../../flutter/impeller/compiler/shader_bundle.h FILE: ../../../flutter/impeller/compiler/shader_lib/flutter/runtime_effect.glsl From 9e3fccc1d889d5099b4c8f0cf7717b22f302736f Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Tue, 12 Dec 2023 15:14:11 -0800 Subject: [PATCH 24/27] Jonah review --- impeller/compiler/reflector.cc | 12 ++++++------ impeller/compiler/shader_bundle.cc | 2 -- impeller/compiler/switches.cc | 6 ++---- impeller/compiler/types.cc | 7 +++---- impeller/compiler/types.h | 2 +- impeller/compiler/utilities.cc | 10 +++++++++- impeller/compiler/utilities.h | 4 +++- lib/gpu/lib/src/shader_library.dart | 2 +- 8 files changed, 25 insertions(+), 20 deletions(-) diff --git a/impeller/compiler/reflector.cc b/impeller/compiler/reflector.cc index 0c617a57529f6..3afb13dab3924 100644 --- a/impeller/compiler/reflector.cc +++ b/impeller/compiler/reflector.cc @@ -352,7 +352,7 @@ std::shared_ptr Reflector::GenerateRuntimeStageData() const { data->AddUniformDescription(std::move(uniform_description)); } - // We only need to worry about reflecting vertex attributes. + // We only need to worry about storing vertex attributes. if (entrypoints.front().execution_model == spv::ExecutionModelVertex) { const auto inputs = compiler_->get_shader_resources().stage_inputs; auto input_offsets = ComputeOffsets(inputs); @@ -1155,7 +1155,7 @@ std::vector Reflector::ReflectBindPrototypes( for (const auto& uniform_buffer : resources.uniform_buffers) { auto& proto = prototypes.emplace_back(BindPrototype{}); proto.return_type = "bool"; - proto.name = ConvertToCamelCase(uniform_buffer.name); + proto.name = ToCamelCase(uniform_buffer.name); { std::stringstream stream; stream << "Bind uniform buffer for resource named " << uniform_buffer.name @@ -1174,7 +1174,7 @@ std::vector Reflector::ReflectBindPrototypes( for (const auto& storage_buffer : resources.storage_buffers) { auto& proto = prototypes.emplace_back(BindPrototype{}); proto.return_type = "bool"; - proto.name = ConvertToCamelCase(storage_buffer.name); + proto.name = ToCamelCase(storage_buffer.name); { std::stringstream stream; stream << "Bind storage buffer for resource named " << storage_buffer.name @@ -1193,7 +1193,7 @@ std::vector Reflector::ReflectBindPrototypes( for (const auto& sampled_image : resources.sampled_images) { auto& proto = prototypes.emplace_back(BindPrototype{}); proto.return_type = "bool"; - proto.name = ConvertToCamelCase(sampled_image.name); + proto.name = ToCamelCase(sampled_image.name); { std::stringstream stream; stream << "Bind combined image sampler for resource named " @@ -1216,7 +1216,7 @@ std::vector Reflector::ReflectBindPrototypes( for (const auto& separate_image : resources.separate_images) { auto& proto = prototypes.emplace_back(BindPrototype{}); proto.return_type = "bool"; - proto.name = ConvertToCamelCase(separate_image.name); + proto.name = ToCamelCase(separate_image.name); { std::stringstream stream; stream << "Bind separate image for resource named " << separate_image.name @@ -1235,7 +1235,7 @@ std::vector Reflector::ReflectBindPrototypes( for (const auto& separate_sampler : resources.separate_samplers) { auto& proto = prototypes.emplace_back(BindPrototype{}); proto.return_type = "bool"; - proto.name = ConvertToCamelCase(separate_sampler.name); + proto.name = ToCamelCase(separate_sampler.name); { std::stringstream stream; stream << "Bind separate sampler for resource named " diff --git a/impeller/compiler/shader_bundle.cc b/impeller/compiler/shader_bundle.cc index 369d1bc588b8a..d71d3bfa3c3a1 100644 --- a/impeller/compiler/shader_bundle.cc +++ b/impeller/compiler/shader_bundle.cc @@ -136,8 +136,6 @@ static std::unique_ptr GenerateShaderFB( return nullptr; } - // TODO(bdero): Vertex attributes. - return result; } diff --git a/impeller/compiler/switches.cc b/impeller/compiler/switches.cc index dbaf911df17b0..46a65c246f294 100644 --- a/impeller/compiler/switches.cc +++ b/impeller/compiler/switches.cc @@ -144,10 +144,8 @@ Switches::Switches(const fml::CommandLine& command_line) use_half_textures(command_line.HasOption("use-half-textures")), require_framebuffer_fetch( command_line.HasOption("require-framebuffer-fetch")) { - auto language = - command_line.GetOptionValueWithDefault("source-language", "glsl"); - std::transform(language.begin(), language.end(), language.begin(), - [](char x) { return std::tolower(x); }); + auto language = ToLowerCase( + command_line.GetOptionValueWithDefault("source-language", "glsl")); source_language = ToSourceLanguage(language); diff --git a/impeller/compiler/types.cc b/impeller/compiler/types.cc index 685122bfb8ae8..2368bcd20fbec 100644 --- a/impeller/compiler/types.cc +++ b/impeller/compiler/types.cc @@ -44,9 +44,7 @@ SourceType SourceTypeFromFileName(const std::string& file_name) { } SourceType SourceTypeFromString(std::string name) { - for (auto it = name.begin(); it != name.end(); it++) { - *it = std::tolower(static_cast(*it)); - } + name = ToLowerCase(name); if (name == "vertex") { return SourceType::kVertexShader; @@ -66,7 +64,8 @@ SourceType SourceTypeFromString(std::string name) { SourceLanguage ToSourceLanguage(const std::string& source_language) { if (source_language == "glsl") { return SourceLanguage::kGLSL; - } else if (source_language == "hlsl") { + } + if (source_language == "hlsl") { return SourceLanguage::kHLSL; } return SourceLanguage::kUnknown; diff --git a/impeller/compiler/types.h b/impeller/compiler/types.h index 2f14cc701a607..dee14a5cdfcd8 100644 --- a/impeller/compiler/types.h +++ b/impeller/compiler/types.h @@ -51,7 +51,7 @@ struct ShaderConfig { std::string entry_point; }; -using ShaderBundleConfig = std::map; +using ShaderBundleConfig = std::unordered_map; bool TargetPlatformIsMetal(TargetPlatform platform); diff --git a/impeller/compiler/utilities.cc b/impeller/compiler/utilities.cc index 8de72a1b24765..ef24dc1681a5d 100644 --- a/impeller/compiler/utilities.cc +++ b/impeller/compiler/utilities.cc @@ -4,6 +4,7 @@ #include "impeller/compiler/utilities.h" +#include #include #include #include @@ -35,7 +36,7 @@ std::string InferShaderNameFromPath(std::string_view path) { return Utf8FromPath(p); } -std::string ConvertToCamelCase(std::string_view string) { +std::string ToCamelCase(std::string_view string) { if (string.empty()) { return ""; } @@ -58,6 +59,13 @@ std::string ConvertToCamelCase(std::string_view string) { return stream.str(); } +std::string ToLowerCase(std::string_view string) { + std::string result = std::string(string); + std::transform(result.begin(), result.end(), result.begin(), + [](char x) { return std::tolower(x); }); + return result; +} + std::string ConvertToEntrypointName(std::string_view string) { if (string.empty()) { return ""; diff --git a/impeller/compiler/utilities.h b/impeller/compiler/utilities.h index ba1145f128521..2aff1530fa2fd 100644 --- a/impeller/compiler/utilities.h +++ b/impeller/compiler/utilities.h @@ -24,7 +24,9 @@ std::string Utf8FromPath(const std::filesystem::path& path); std::string InferShaderNameFromPath(std::string_view path); -std::string ConvertToCamelCase(std::string_view string); +std::string ToCamelCase(std::string_view string); + +std::string ToLowerCase(std::string_view string); /// @brief Ensure that the entrypoint name is a valid identifier in the target /// language. diff --git a/lib/gpu/lib/src/shader_library.dart b/lib/gpu/lib/src/shader_library.dart index 636229d7042a5..d9df64e73c875 100644 --- a/lib/gpu/lib/src/shader_library.dart +++ b/lib/gpu/lib/src/shader_library.dart @@ -21,7 +21,7 @@ base class ShaderLibrary extends NativeFieldWrapperClass1 { // Hold a Dart-side reference to shaders in the library as they're wrapped for // the first time. This prevents the wrapper from getting prematurely // destroyed. - Map shaders_ = {}; + final Map shaders_ = {}; Shader? operator [](String shaderName) { // This `flutter_gpu` library isn't always registered as part of the builtin From 19199224b0758890e6abf109fcb1d56c5bc088a5 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Tue, 12 Dec 2023 23:38:38 -0800 Subject: [PATCH 25/27] Zach review --- impeller/compiler/reflector.cc | 2 +- impeller/fixtures/BUILD.gn | 4 ++-- impeller/tools/impeller.gni | 41 +++++++++++++++++++++++++--------- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/impeller/compiler/reflector.cc b/impeller/compiler/reflector.cc index 3afb13dab3924..05827e341f3ef 100644 --- a/impeller/compiler/reflector.cc +++ b/impeller/compiler/reflector.cc @@ -414,7 +414,7 @@ std::shared_ptr Reflector::InflateTemplate( env.set_lstrip_blocks(true); env.add_callback("camel_case", 1u, [](inja::Arguments& args) { - return ConvertToCamelCase(args.at(0u)->get()); + return ToCamelCase(args.at(0u)->get()); }); env.add_callback("to_shader_stage", 1u, [](inja::Arguments& args) { diff --git a/impeller/fixtures/BUILD.gn b/impeller/fixtures/BUILD.gn index 19d003344fc36..1867b65862e0d 100644 --- a/impeller/fixtures/BUILD.gn +++ b/impeller/fixtures/BUILD.gn @@ -76,10 +76,10 @@ test_fixtures("file_fixtures") { "blue_noise.png", "boston.jpg", "embarcadero.jpg", - "flutter_gpu_unlit.frag", - "flutter_gpu_unlit.vert", "flutter_gpu_texture.frag", "flutter_gpu_texture.vert", + "flutter_gpu_unlit.frag", + "flutter_gpu_unlit.vert", "flutter_logo_baked.glb", "kalimba.jpg", "multiple_stages.hlsl", diff --git a/impeller/tools/impeller.gni b/impeller/tools/impeller.gni index 7e9d8c06e70ee..a07963ee380b3 100644 --- a/impeller/tools/impeller.gni +++ b/impeller/tools/impeller.gni @@ -239,8 +239,12 @@ template("embed_blob") { } # Dispatches to the build or prebuilt impellerc depending on the value of -# the impeller_use_prebuilt_impellerc argument. Forwards all variables to -# compiled_action_foreach or action_foreach as appropriate. +# the `impeller_use_prebuilt_impellerc` argument. +# * When the `single_invocation` argument is false, all variables are +# forwarded to `compiled_action_foreach` or `action_foreach`, which will +# invoke `impellerc` separately for each source. +# * When the `single_invocation` argument is true, `impellerc` is only +# invoked once via `compiled_action` or `action`. template("_impellerc") { if (invoker.single_invocation) { if (impeller_use_prebuilt_impellerc == "") { @@ -275,6 +279,30 @@ template("_impellerc") { } } +# Required: shaders The list of shaders inputs to compile. +# Required: shader_target_flag The target flag to append. Valid options: +# --sksl +# --metal-ios +# --metal-desktop +# --opengl-es +# --opengl-desktop +# --vulkan +# --runtime-stage-metal +# Required: sl_file_extension The file extension to use for output files. +# Not required for --shader_bundle mode. +# Optional: iplr Causes --sl output to be in iplr/runtime +# stage flatbuffer format. +# Optional: shader_bundle Specifies a Flutter GPU shader bundle +# configuration. +# Required: shader_bundle_output Specifies the output filename of the shader +# bundle. This is only required if +# shader_bundle is supplied. +# Optional: defines Specifies a list of valueless macro +# definitions. +# Optional: intermediates_subdir Specifies the subdirectory in which to put +# intermediates. +# Optional: json Causes output format to be JSON instead of +# flatbuffer. template("impellerc") { assert(defined(invoker.shaders), "Impeller shaders must be specified.") assert(defined(invoker.shader_target_flag), @@ -305,15 +333,6 @@ template("impellerc") { "shader_bundle_output", ]) - # Optional: invoker.iplr Causes --sl output to be in iplr format. - # Optional: invoker.shader_bundle specifies a Flutter GPU shader bundle configuration. - # Optional: invoker.shader_bundle_output specifies the output filename of the shader - # bundle. This is required if invoker.shader_bundle is supplied. - # Optional: invoker.defines specifies a list of valueless macro definitions. - # Optional: invoker.intermediates_subdir specifies the subdirectory in which - # to put intermediates. - # Optional: invoker.json Causes output format to be JSON instead of flatbuffer. - _impellerc(target_name) { shader_bundle = defined(invoker.shader_bundle) From d7ef71dd670771bd7ed3ca61f8064a3d9c0d11d5 Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Tue, 12 Dec 2023 23:41:15 -0800 Subject: [PATCH 26/27] Jonah review --- lib/gpu/lib/src/shader_library.dart | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/lib/gpu/lib/src/shader_library.dart b/lib/gpu/lib/src/shader_library.dart index d9df64e73c875..859ba02d38f29 100644 --- a/lib/gpu/lib/src/shader_library.dart +++ b/lib/gpu/lib/src/shader_library.dart @@ -21,7 +21,7 @@ base class ShaderLibrary extends NativeFieldWrapperClass1 { // Hold a Dart-side reference to shaders in the library as they're wrapped for // the first time. This prevents the wrapper from getting prematurely // destroyed. - final Map shaders_ = {}; + final Map shaders_ = {}; Shader? operator [](String shaderName) { // This `flutter_gpu` library isn't always registered as part of the builtin @@ -30,14 +30,8 @@ base class ShaderLibrary extends NativeFieldWrapperClass1 { // Providing a new wrapper to [_getShader] for wrapping the native // counterpart (if it hasn't been wrapped already) is a hack to work around // this. - Shader? result = shaders_[shaderName]; - if (result == null) { - result = _getShader(shaderName, Shader._()); - if (result != null) { - shaders_[shaderName] = result; - } - } - return result; + return shaders_.putIfAbsent( + shaderName, () => _getShader(shaderName, Shader._())); } @Native( From be1078c9a9ef14995e2cf5647fb35032d29484ad Mon Sep 17 00:00:00 2001 From: Brandon DeRosier Date: Wed, 13 Dec 2023 15:52:09 -0800 Subject: [PATCH 27/27] zach review --- impeller/compiler/BUILD.gn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impeller/compiler/BUILD.gn b/impeller/compiler/BUILD.gn index 1debfcbf71d54..3f508fe605f1a 100644 --- a/impeller/compiler/BUILD.gn +++ b/impeller/compiler/BUILD.gn @@ -68,9 +68,9 @@ impeller_component("compiler_lib") { "../geometry", "../runtime_stage", "//flutter/fml", + "//flutter/impeller/shader_bundle:shader_bundle_flatbuffers", # All third_party deps must be included by the global license script. - "//flutter/impeller/shader_bundle:shader_bundle_flatbuffers", "//third_party/inja", "//third_party/shaderc_flutter", "//third_party/spirv_cross_flutter",