diff --git a/mlir/docs/DialectConversion.md b/mlir/docs/DialectConversion.md index 556e73c2d56c7..7070351755e7a 100644 --- a/mlir/docs/DialectConversion.md +++ b/mlir/docs/DialectConversion.md @@ -280,6 +280,15 @@ target types. If the source type is converted to itself, we say it is a "legal" type. Type conversions are specified via the `addConversion` method described below. +There are two kind of conversion functions: context-aware and context-unaware +conversions. A context-unaware conversion function converts a `Type` into a +`Type`. A context-aware conversion function converts a `Value` into a type. The +latter allows users to customize type conversion rules based on the IR. + +Note: When there is at least one context-aware type conversion function, the +result of type conversions can no longer be cached, which can increase +compilation time. Use this feature with caution! + A `materialization` describes how a list of values should be converted to a list of values with specific types. An important distinction from a `conversion` is that a `materialization` can produce IR, whereas a `conversion` @@ -332,29 +341,31 @@ Several of the available hooks are detailed below: ```c++ class TypeConverter { public: - /// Register a conversion function. A conversion function defines how a given - /// source type should be converted. A conversion function must be convertible - /// to any of the following forms(where `T` is a class derived from `Type`: - /// * Optional(T) + /// Register a conversion function. A conversion function must be convertible + /// to any of the following forms (where `T` is `Value` or a class derived + /// from `Type`, including `Type` itself): + /// + /// * std::optional(T) /// - This form represents a 1-1 type conversion. It should return nullptr - /// or `std::nullopt` to signify failure. If `std::nullopt` is returned, the - /// converter is allowed to try another conversion function to perform - /// the conversion. - /// * Optional(T, SmallVectorImpl &) + /// or `std::nullopt` to signify failure. If `std::nullopt` is returned, + /// the converter is allowed to try another conversion function to + /// perform the conversion. + /// * std::optional(T, SmallVectorImpl &) /// - This form represents a 1-N type conversion. It should return - /// `failure` or `std::nullopt` to signify a failed conversion. If the new - /// set of types is empty, the type is removed and any usages of the + /// `failure` or `std::nullopt` to signify a failed conversion. If the + /// new set of types is empty, the type is removed and any usages of the /// existing value are expected to be removed during conversion. If /// `std::nullopt` is returned, the converter is allowed to try another /// conversion function to perform the conversion. - /// * Optional(T, SmallVectorImpl &, ArrayRef) - /// - This form represents a 1-N type conversion supporting recursive - /// types. The first two arguments and the return value are the same as - /// for the regular 1-N form. The third argument is contains is the - /// "call stack" of the recursive conversion: it contains the list of - /// types currently being converted, with the current type being the - /// last one. If it is present more than once in the list, the - /// conversion concerns a recursive type. + /// + /// Conversion functions that accept `Value` as the first argument are + /// context-aware. I.e., they can take into account IR when converting the + /// type of the given value. Context-unaware conversion functions accept + /// `Type` or a derived class as the first argument. + /// + /// Note: Context-unaware conversions are cached, but context-aware + /// conversions are not. + /// /// Note: When attempting to convert a type, e.g. via 'convertType', the /// mostly recently added conversions will be invoked first. template (T) /// - This form represents a 1-1 type conversion. It should return nullptr @@ -154,6 +155,14 @@ class TypeConverter { /// `std::nullopt` is returned, the converter is allowed to try another /// conversion function to perform the conversion. /// + /// Conversion functions that accept `Value` as the first argument are + /// context-aware. I.e., they can take into account IR when converting the + /// type of the given value. Context-unaware conversion functions accept + /// `Type` or a derived class as the first argument. + /// + /// Note: Context-unaware conversions are cached, but context-aware + /// conversions are not. + /// /// Note: When attempting to convert a type, e.g. via 'convertType', the /// mostly recently added conversions will be invoked first. template (std::forward(callback))); } - /// Convert the given type. This function should return failure if no valid + /// Convert the given type. This function returns failure if no valid /// conversion exists, success otherwise. If the new set of types is empty, /// the type is removed and any usages of the existing value are expected to /// be removed during conversion. + /// + /// Note: This overload invokes only context-unaware type conversion + /// functions. Users should call the other overload if possible. LogicalResult convertType(Type t, SmallVectorImpl &results) const; + /// Convert the type of the given value. This function returns failure if no + /// valid conversion exists, success otherwise. If the new set of types is + /// empty, the type is removed and any usages of the existing value are + /// expected to be removed during conversion. + /// + /// Note: This overload invokes both context-aware and context-unaware type + /// conversion functions. + LogicalResult convertType(Value v, SmallVectorImpl &results) const; + /// This hook simplifies defining 1-1 type conversions. This function returns /// the type to convert to on success, and a null type on failure. Type convertType(Type t) const; + Type convertType(Value v) const; /// Attempts a 1-1 type conversion, expecting the result type to be /// `TargetType`. Returns the converted type cast to `TargetType` on success, @@ -259,25 +281,36 @@ class TypeConverter { TargetType convertType(Type t) const { return dyn_cast_or_null(convertType(t)); } + template + TargetType convertType(Value v) const { + return dyn_cast_or_null(convertType(v)); + } - /// Convert the given set of types, filling 'results' as necessary. This - /// returns failure if the conversion of any of the types fails, success + /// Convert the given types, filling 'results' as necessary. This returns + /// "failure" if the conversion of any of the types fails, "success" /// otherwise. LogicalResult convertTypes(TypeRange types, SmallVectorImpl &results) const; + /// Convert the types of the given values, filling 'results' as necessary. + /// This returns "failure" if the conversion of any of the types fails, + /// "success" otherwise. + LogicalResult convertTypes(ValueRange values, + SmallVectorImpl &results) const; + /// Return true if the given type is legal for this type converter, i.e. the /// type converts to itself. bool isLegal(Type type) const; + bool isLegal(Value value) const; /// Return true if all of the given types are legal for this type converter. - template - std::enable_if_t::value && - !std::is_convertible::value, - bool> - isLegal(RangeT &&range) const { + bool isLegal(TypeRange range) const { return llvm::all_of(range, [this](Type type) { return isLegal(type); }); } + bool isLegal(ValueRange range) const { + return llvm::all_of(range, [this](Value value) { return isLegal(value); }); + } + /// Return true if the given operation has legal operand and result types. bool isLegal(Operation *op) const; @@ -296,6 +329,11 @@ class TypeConverter { LogicalResult convertSignatureArgs(TypeRange types, SignatureConversion &result, unsigned origInputOffset = 0) const; + LogicalResult convertSignatureArg(unsigned inputNo, Value value, + SignatureConversion &result) const; + LogicalResult convertSignatureArgs(ValueRange values, + SignatureConversion &result, + unsigned origInputOffset = 0) const; /// This function converts the type signature of the given block, by invoking /// 'convertSignatureArg' for each argument. This function should return a @@ -329,7 +367,7 @@ class TypeConverter { /// types is empty, the type is removed and any usages of the existing value /// are expected to be removed during conversion. using ConversionCallbackFn = std::function( - Type, SmallVectorImpl &)>; + PointerUnion, SmallVectorImpl &)>; /// The signature of the callback used to materialize a source conversion. /// @@ -349,13 +387,14 @@ class TypeConverter { /// Generate a wrapper for the given callback. This allows for accepting /// different callback forms, that all compose into a single version. - /// With callback of form: `std::optional(T)` + /// With callback of form: `std::optional(T)`, where `T` can be a + /// `Value` or a `Type` (or a class derived from `Type`). template std::enable_if_t, ConversionCallbackFn> - wrapCallback(FnT &&callback) const { + wrapCallback(FnT &&callback) { return wrapCallback([callback = std::forward(callback)]( - T type, SmallVectorImpl &results) { - if (std::optional resultOpt = callback(type)) { + T typeOrValue, SmallVectorImpl &results) { + if (std::optional resultOpt = callback(typeOrValue)) { bool wasSuccess = static_cast(*resultOpt); if (wasSuccess) results.push_back(*resultOpt); @@ -365,20 +404,49 @@ class TypeConverter { }); } /// With callback of form: `std::optional( - /// T, SmallVectorImpl &, ArrayRef)`. + /// T, SmallVectorImpl &)`, where `T` is a type. template - std::enable_if_t &>, + std::enable_if_t &> && + std::is_base_of_v, ConversionCallbackFn> wrapCallback(FnT &&callback) const { return [callback = std::forward(callback)]( - Type type, + PointerUnion typeOrValue, SmallVectorImpl &results) -> std::optional { - T derivedType = dyn_cast(type); + T derivedType; + if (Type t = dyn_cast(typeOrValue)) { + derivedType = dyn_cast(t); + } else if (Value v = dyn_cast(typeOrValue)) { + derivedType = dyn_cast(v.getType()); + } else { + llvm_unreachable("unexpected variant"); + } if (!derivedType) return std::nullopt; return callback(derivedType, results); }; } + /// With callback of form: `std::optional( + /// T, SmallVectorImpl)`, where `T` is a `Value`. + template + std::enable_if_t &> && + std::is_same_v, + ConversionCallbackFn> + wrapCallback(FnT &&callback) { + hasContextAwareTypeConversions = true; + return [callback = std::forward(callback)]( + PointerUnion typeOrValue, + SmallVectorImpl &results) -> std::optional { + if (Type t = dyn_cast(typeOrValue)) { + // Context-aware type conversion was called with a type. + return std::nullopt; + } else if (Value v = dyn_cast(typeOrValue)) { + return callback(v, results); + } + llvm_unreachable("unexpected variant"); + return std::nullopt; + }; + } /// Register a type conversion. void registerConversion(ConversionCallbackFn callback) { @@ -505,6 +573,12 @@ class TypeConverter { mutable DenseMap> cachedMultiConversions; /// A mutex used for cache access mutable llvm::sys::SmartRWMutex cacheMutex; + /// Whether the type converter has context-aware type conversions. I.e., + /// conversion rules that depend on the SSA value instead of just the type. + /// Type conversion caching is deactivated when there are context-aware + /// conversions because the type converter may return different results for + /// the same input type. + bool hasContextAwareTypeConversions = false; }; //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp index 3b75970c98ad4..072bc501aa5c6 100644 --- a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp +++ b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp @@ -52,8 +52,8 @@ class Structural1ToNConversionPattern : public OpConversionPattern { SmallVector offsets; offsets.push_back(0); // Do the type conversion and record the offsets. - for (Type type : op.getResultTypes()) { - if (failed(typeConverter->convertTypes(type, dstTypes))) + for (Value v : op.getResults()) { + if (failed(typeConverter->convertType(v, dstTypes))) return rewriter.notifyMatchFailure(op, "could not convert result type"); offsets.push_back(dstTypes.size()); } @@ -127,7 +127,6 @@ class ConvertForOpTypes // Inline the type converted region from the original operation. rewriter.inlineRegionBefore(op.getRegion(), newOp.getRegion(), newOp.getRegion().end()); - return newOp; } }; @@ -226,15 +225,14 @@ void mlir::scf::populateSCFStructuralTypeConversions( void mlir::scf::populateSCFStructuralTypeConversionTarget( const TypeConverter &typeConverter, ConversionTarget &target) { - target.addDynamicallyLegalOp([&](Operation *op) { - return typeConverter.isLegal(op->getResultTypes()); - }); + target.addDynamicallyLegalOp( + [&](Operation *op) { return typeConverter.isLegal(op->getResults()); }); target.addDynamicallyLegalOp([&](scf::YieldOp op) { // We only have conversions for a subset of ops that use scf.yield // terminators. if (!isa(op->getParentOp())) return true; - return typeConverter.isLegal(op.getOperandTypes()); + return typeConverter.isLegal(op.getOperands()); }); target.addDynamicallyLegalOp( [&](Operation *op) { return typeConverter.isLegal(op); }); diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp index e3248204d6694..a0232937e9a78 100644 --- a/mlir/lib/Transforms/Utils/DialectConversion.cpp +++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp @@ -1436,7 +1436,7 @@ LogicalResult ConversionPatternRewriterImpl::remapValues( // If there is no legal conversion, fail to match this pattern. SmallVector legalTypes; - if (failed(currentTypeConverter->convertType(origType, legalTypes))) { + if (failed(currentTypeConverter->convertType(operand, legalTypes))) { notifyMatchFailure(operandLoc, [=](Diagnostic &diag) { diag << "unable to convert type for " << valueDiagTag << " #" << it.index() << ", type was " << origType; @@ -3430,6 +3430,27 @@ LogicalResult TypeConverter::convertType(Type t, return failure(); } +LogicalResult TypeConverter::convertType(Value v, + SmallVectorImpl &results) const { + assert(v && "expected non-null value"); + + // If this type converter does not have context-aware type conversions, call + // the type-based overload, which has caching. + if (!hasContextAwareTypeConversions) + return convertType(v.getType(), results); + + // Walk the added converters in reverse order to apply the most recently + // registered first. + for (const ConversionCallbackFn &converter : llvm::reverse(conversions)) { + if (std::optional result = converter(v, results)) { + if (!succeeded(*result)) + return failure(); + return success(); + } + } + return failure(); +} + Type TypeConverter::convertType(Type t) const { // Use the multi-type result version to convert the type. SmallVector results; @@ -3440,6 +3461,16 @@ Type TypeConverter::convertType(Type t) const { return results.size() == 1 ? results.front() : nullptr; } +Type TypeConverter::convertType(Value v) const { + // Use the multi-type result version to convert the type. + SmallVector results; + if (failed(convertType(v, results))) + return nullptr; + + // Check to ensure that only one type was produced. + return results.size() == 1 ? results.front() : nullptr; +} + LogicalResult TypeConverter::convertTypes(TypeRange types, SmallVectorImpl &results) const { @@ -3449,21 +3480,38 @@ TypeConverter::convertTypes(TypeRange types, return success(); } +LogicalResult +TypeConverter::convertTypes(ValueRange values, + SmallVectorImpl &results) const { + for (Value value : values) + if (failed(convertType(value, results))) + return failure(); + return success(); +} + bool TypeConverter::isLegal(Type type) const { return convertType(type) == type; } + +bool TypeConverter::isLegal(Value value) const { + return convertType(value) == value.getType(); +} + bool TypeConverter::isLegal(Operation *op) const { - return isLegal(op->getOperandTypes()) && isLegal(op->getResultTypes()); + return isLegal(op->getOperands()) && isLegal(op->getResults()); } bool TypeConverter::isLegal(Region *region) const { - return llvm::all_of(*region, [this](Block &block) { - return isLegal(block.getArgumentTypes()); - }); + return llvm::all_of( + *region, [this](Block &block) { return isLegal(block.getArguments()); }); } bool TypeConverter::isSignatureLegal(FunctionType ty) const { - return isLegal(llvm::concat(ty.getInputs(), ty.getResults())); + if (!isLegal(ty.getInputs())) + return false; + if (!isLegal(ty.getResults())) + return false; + return true; } LogicalResult @@ -3491,6 +3539,31 @@ TypeConverter::convertSignatureArgs(TypeRange types, return failure(); return success(); } +LogicalResult +TypeConverter::convertSignatureArg(unsigned inputNo, Value value, + SignatureConversion &result) const { + // Try to convert the given input type. + SmallVector convertedTypes; + if (failed(convertType(value, convertedTypes))) + return failure(); + + // If this argument is being dropped, there is nothing left to do. + if (convertedTypes.empty()) + return success(); + + // Otherwise, add the new inputs. + result.addInputs(inputNo, convertedTypes); + return success(); +} +LogicalResult +TypeConverter::convertSignatureArgs(ValueRange values, + SignatureConversion &result, + unsigned origInputOffset) const { + for (unsigned i = 0, e = values.size(); i != e; ++i) + if (failed(convertSignatureArg(origInputOffset + i, values[i], result))) + return failure(); + return success(); +} Value TypeConverter::materializeSourceConversion(OpBuilder &builder, Location loc, Type resultType, @@ -3534,7 +3607,7 @@ SmallVector TypeConverter::materializeTargetConversion( std::optional TypeConverter::convertBlockSignature(Block *block) const { SignatureConversion conversion(block->getNumArguments()); - if (failed(convertSignatureArgs(block->getArgumentTypes(), conversion))) + if (failed(convertSignatureArgs(block->getArguments(), conversion))) return std::nullopt; return conversion; } @@ -3659,7 +3732,7 @@ mlir::convertOpResultTypes(Operation *op, ValueRange operands, newOp.addOperands(operands); SmallVector newResultTypes; - if (failed(converter.convertTypes(op->getResultTypes(), newResultTypes))) + if (failed(converter.convertTypes(op->getResults(), newResultTypes))) return rewriter.notifyMatchFailure(loc, "couldn't convert return types"); newOp.addTypes(newResultTypes); diff --git a/mlir/test/Transforms/test-context-aware-type-converter.mlir b/mlir/test/Transforms/test-context-aware-type-converter.mlir new file mode 100644 index 0000000000000..ae178b676a392 --- /dev/null +++ b/mlir/test/Transforms/test-context-aware-type-converter.mlir @@ -0,0 +1,40 @@ +// RUN: mlir-opt %s -test-legalize-type-conversion="allow-pattern-rollback=0" -split-input-file -verify-diagnostics | FileCheck %s + +// CHECK-LABEL: func @simple_context_aware_conversion_1() +func.func @simple_context_aware_conversion_1() attributes {increment = 1 : i64} { + // Case 1: Convert i37 --> i38. + // CHECK: %[[cast:.*]] = builtin.unrealized_conversion_cast %{{.*}} : i37 to i38 + // CHECK: "test.legal_op_d"(%[[cast]]) : (i38) -> () + %0 = "test.context_op"() : () -> (i37) + "test.replace_with_legal_op"(%0) : (i37) -> () + return +} + +// CHECK-LABEL: func @simple_context_aware_conversion_2() +func.func @simple_context_aware_conversion_2() attributes {increment = 2 : i64} { + // Case 2: Convert i37 --> i39. + // CHECK: %[[cast:.*]] = builtin.unrealized_conversion_cast %{{.*}} : i37 to i39 + // CHECK: "test.legal_op_d"(%[[cast]]) : (i39) -> () + %0 = "test.context_op"() : () -> (i37) + "test.replace_with_legal_op"(%0) : (i37) -> () + return +} + +// ----- + +// Note: This test case does not work with allow-pattern-rollback=1. When +// rollback is enabled, the type converter cannot find the enclosing function +// because the operand of the scf.yield is pointing to a detached block. + +// CHECK-LABEL: func @convert_block_arguments +// CHECK: %[[cast:.*]] = builtin.unrealized_conversion_cast %{{.*}} : i37 to i38 +// CHECK: scf.for %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%[[iter:.*]] = %[[cast]]) -> (i38) { +// CHECK: scf.yield %[[iter]] : i38 +// CHECK: } +func.func @convert_block_arguments(%lb: index, %ub: index, %step: index) attributes {increment = 1 : i64} { + %0 = "test.context_op"() : () -> (i37) + scf.for %iv = %lb to %ub step %step iter_args(%arg0 = %0) -> i37 { + scf.yield %arg0 : i37 + } + return +} diff --git a/mlir/test/Transforms/test-legalize-type-conversion.mlir b/mlir/test/Transforms/test-legalize-type-conversion.mlir index 9bffe92b374d5..c003f8b2cb1cd 100644 --- a/mlir/test/Transforms/test-legalize-type-conversion.mlir +++ b/mlir/test/Transforms/test-legalize-type-conversion.mlir @@ -142,3 +142,4 @@ func.func @test_signature_conversion_no_converter() { }) : () -> () return } + diff --git a/mlir/test/lib/Dialect/Test/TestPatterns.cpp b/mlir/test/lib/Dialect/Test/TestPatterns.cpp index b6f16ac1b5c48..95f381ec471d6 100644 --- a/mlir/test/lib/Dialect/Test/TestPatterns.cpp +++ b/mlir/test/lib/Dialect/Test/TestPatterns.cpp @@ -13,6 +13,7 @@ #include "mlir/Dialect/CommonFolders.h" #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/Dialect/Func/Transforms/FuncConversions.h" +#include "mlir/Dialect/SCF/Transforms/Patterns.h" #include "mlir/Dialect/Tensor/IR/Tensor.h" #include "mlir/IR/BuiltinAttributes.h" #include "mlir/IR/Matchers.h" @@ -1983,9 +1984,9 @@ struct TestReplaceWithLegalOp : public ConversionPattern { : ConversionPattern(converter, "test.replace_with_legal_op", /*benefit=*/1, ctx) {} LogicalResult - matchAndRewrite(Operation *op, ArrayRef operands, + matchAndRewrite(Operation *op, ArrayRef operands, ConversionPatternRewriter &rewriter) const final { - rewriter.replaceOpWithNewOp(op, operands[0]); + rewriter.replaceOpWithNewOp(op, operands[0].front()); return success(); } }; @@ -1994,6 +1995,10 @@ struct TestTypeConversionDriver : public PassWrapper> { MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestTypeConversionDriver) + TestTypeConversionDriver() = default; + TestTypeConversionDriver(const TestTypeConversionDriver &other) + : PassWrapper(other) {} + void getDependentDialects(DialectRegistry ®istry) const override { registry.insert(); } @@ -2020,8 +2025,13 @@ struct TestTypeConversionDriver // Otherwise, the type is illegal. return nullptr; }); - converter.addConversion([](IntegerType type, SmallVectorImpl &) { - // Drop all integer types. + converter.addConversion([](IndexType type) { return type; }); + converter.addConversion([](IntegerType type, SmallVectorImpl &types) { + if (type.isInteger(38)) { + // i38 is legal. + types.push_back(type); + } + // Drop all other integer types. return success(); }); converter.addConversion( @@ -2058,6 +2068,33 @@ struct TestTypeConversionDriver results.push_back(result); return success(); }); + converter.addConversion([](Value v) -> std::optional { + // Context-aware type conversion rule that converts i37 to + // i(37 + increment). The increment is taken from the enclosing + // function. + auto intType = dyn_cast(v.getType()); + if (!intType || intType.getWidth() != 37) + return std::nullopt; + Region *r = v.getParentRegion(); + if (!r) { + // No enclosing region found. This can happen when running with + // allow-pattern-rollback = true. Context-aware type conversions are + // not fully supported when running in rollback mode. + return Type(); + } + Operation *op = r->getParentOp(); + if (!op) + return Type(); + if (!isa(op)) + op = op->getParentOfType(); + if (!op) + return Type(); + auto incrementAttr = op->getAttrOfType("increment"); + if (!incrementAttr) + return Type(); + return IntegerType::get(v.getContext(), + intType.getWidth() + incrementAttr.getInt()); + }); /// Add the legal set of type materializations. converter.addSourceMaterialization([](OpBuilder &builder, Type resultType, @@ -2078,9 +2115,19 @@ struct TestTypeConversionDriver // Otherwise, fail. return nullptr; }); + // Materialize i37 to any desired type with unrealized_conversion_cast. + converter.addTargetMaterialization([](OpBuilder &builder, Type type, + ValueRange inputs, + Location loc) -> Value { + if (inputs.size() != 1 || !inputs[0].getType().isInteger(37)) + return Value(); + return builder.create(loc, type, inputs) + .getResult(0); + }); // Initialize the conversion target. mlir::ConversionTarget target(getContext()); + target.addLegalOp(OperationName("test.context_op", &getContext())); target.addLegalOp(); target.addDynamicallyLegalOp([](TestTypeProducerOp op) { auto recursiveType = dyn_cast(op.getType()); @@ -2111,11 +2158,19 @@ struct TestTypeConversionDriver patterns.add(&getContext()); mlir::populateAnyFunctionOpInterfaceTypeConversionPattern(patterns, converter); + mlir::scf::populateSCFStructuralTypeConversionsAndLegality( + converter, patterns, target); + ConversionConfig config; + config.allowPatternRollback = allowPatternRollback; if (failed(applyPartialConversion(getOperation(), target, - std::move(patterns)))) + std::move(patterns), config))) signalPassFailure(); } + + Option allowPatternRollback{*this, "allow-pattern-rollback", + llvm::cl::desc("Allow pattern rollback"), + llvm::cl::init(true)}; }; } // namespace