diff --git a/components/yaml/build.gradle.kts b/components/yaml/build.gradle.kts new file mode 100644 index 00000000000..5acc8dd4cc2 --- /dev/null +++ b/components/yaml/build.gradle.kts @@ -0,0 +1,14 @@ +plugins { + id("me.champeau.jmh") +} + +apply(from = "$rootDir/gradle/java.gradle") + +jmh { + version = "1.28" +} + +// https://repo1.maven.org/maven2/org/yaml/snakeyaml/2.4/snakeyaml-2.4.pom +dependencies { + implementation("org.yaml", "snakeyaml", "2.4") +} diff --git a/components/yaml/src/main/java/datadog/yaml/YamlParser.java b/components/yaml/src/main/java/datadog/yaml/YamlParser.java new file mode 100644 index 00000000000..01aa7216fc1 --- /dev/null +++ b/components/yaml/src/main/java/datadog/yaml/YamlParser.java @@ -0,0 +1,19 @@ +package datadog.yaml; + +import java.io.FileInputStream; +import java.io.IOException; +import org.yaml.snakeyaml.Yaml; + +public class YamlParser { + // Supports clazz == null for default yaml parsing + public static T parse(String filePath, Class clazz) throws IOException { + Yaml yaml = new Yaml(); + try (FileInputStream fis = new FileInputStream(filePath)) { + if (clazz == null) { + return yaml.load(fis); + } else { + return yaml.loadAs(fis, clazz); + } + } + } +} diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Constants.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Constants.java index 0ed02193072..d6ddee8410d 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Constants.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Constants.java @@ -16,6 +16,7 @@ public final class Constants { public static final String[] BOOTSTRAP_PACKAGE_PREFIXES = { "datadog.slf4j", "datadog.json", + "datadog.yaml", "datadog.context", "datadog.cli", "datadog.appsec.api", diff --git a/dd-java-agent/build.gradle b/dd-java-agent/build.gradle index 95737b79eeb..d03ea2e440e 100644 --- a/dd-java-agent/build.gradle +++ b/dd-java-agent/build.gradle @@ -59,8 +59,9 @@ ext.generalShadowJarConfig = { final String projectName = "${project.name}" - // Prevents conflict with other OkHttp instances, but don't relocate instrumentation + // Prevents conflict with other instances, but doesn't relocate instrumentation if (!projectName.equals('instrumentation')) { + relocate 'org.yaml.snakeyaml', 'datadog.snakeyaml' relocate 'okhttp3', 'datadog.okhttp3' relocate 'okio', 'datadog.okio' } diff --git a/dd-java-agent/instrumentation/build.gradle b/dd-java-agent/instrumentation/build.gradle index 0cc4e692e3f..3aecf9212b3 100644 --- a/dd-java-agent/instrumentation/build.gradle +++ b/dd-java-agent/instrumentation/build.gradle @@ -59,6 +59,9 @@ subprojects { Project subProj -> def name = "java$version.majorVersion" jdkCompile = "main_${name}Implementation" } + configurations.muzzleBootstrap { + exclude group: 'org.yaml', module : 'snakeyaml' // we vendor this in the agent jar + } dependencies { // Apply common dependencies for instrumentation. implementation project(':dd-trace-api') diff --git a/dd-java-agent/instrumentation/snakeyaml/build.gradle b/dd-java-agent/instrumentation/snakeyaml/build.gradle index 396fe9b937a..5decb514c66 100644 --- a/dd-java-agent/instrumentation/snakeyaml/build.gradle +++ b/dd-java-agent/instrumentation/snakeyaml/build.gradle @@ -18,7 +18,13 @@ addTestSuiteForDir('latestDepTest', 'test') dependencies { compileOnly group: 'org.yaml', name: 'snakeyaml', version: '1.33' - testImplementation group: 'org.yaml', name: 'snakeyaml', version: '1.33' + + testImplementation('org.yaml:snakeyaml') { + version { + strictly "[1.4, 2.0)" + prefer '1.33' + } + } latestDepTestImplementation group: 'org.yaml', name: 'snakeyaml', version: '1.+' } diff --git a/dd-java-agent/testing/build.gradle b/dd-java-agent/testing/build.gradle index a38cbb30819..013fddfbdcb 100644 --- a/dd-java-agent/testing/build.gradle +++ b/dd-java-agent/testing/build.gradle @@ -38,6 +38,10 @@ excludedClassesCoverage += [ 'datadog.trace.agent.test.TestProfilingContextIntegration.TestQueueTiming' ] +configurations.api { + exclude group: 'org.yaml', module: 'snakeyaml' // we vendor this in the agent jar +} + dependencies { api libs.bytebuddy api libs.bytebuddyagent diff --git a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/SpockRunner.java b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/SpockRunner.java index eda9e4d86d4..83bf7732644 100644 --- a/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/SpockRunner.java +++ b/dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/SpockRunner.java @@ -39,6 +39,7 @@ public class SpockRunner extends JUnitPlatform { public static final String[] BOOTSTRAP_PACKAGE_PREFIXES_COPY = { "datadog.slf4j", "datadog.json", + "datadog.yaml", "datadog.context", "datadog.cli", "datadog.appsec.api", diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 76e8061f5d8..4fbdac06696 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -18,6 +18,7 @@ final class CachedData { exclude(project(':communication')) exclude(project(':components:context')) exclude(project(':components:json')) + exclude(project(':components:yaml')) exclude(project(':components:cli')) exclude(project(':remote-config:remote-config-api')) exclude(project(':remote-config:remote-config-core')) @@ -49,6 +50,9 @@ final class CachedData { // cafe_crypto and its transitives exclude(dependency('cafe.cryptography::')) + + // snakeyaml and its transitives + exclude(dependency('org.yaml:snakeyaml')) } ] } diff --git a/internal-api/build.gradle b/internal-api/build.gradle index edaaf1d860f..3ad546c3912 100644 --- a/internal-api/build.gradle +++ b/internal-api/build.gradle @@ -239,6 +239,7 @@ dependencies { api project(':dd-trace-api') api libs.slf4j api project(':components:context') + api project(':components:yaml') api project(':components:cli') api project(":utils:time-utils") @@ -246,6 +247,7 @@ dependencies { // it contains annotations that are also present in the instrumented application classes api "com.datadoghq:dd-javac-plugin-client:0.2.2" + testImplementation("org.yaml:snakeyaml:2.4") testImplementation project(":utils:test-utils") testImplementation("org.assertj:assertj-core:3.20.2") testImplementation libs.bundles.junit5 @@ -253,7 +255,6 @@ dependencies { testImplementation libs.commons.math testImplementation libs.bundles.mockito testImplementation libs.truth - testImplementation 'org.yaml:snakeyaml:2.0' } jmh { diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/StableConfigParser.java b/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/StableConfigParser.java index bb651e11588..0c7554f2e0e 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/StableConfigParser.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/StableConfigParser.java @@ -1,77 +1,179 @@ package datadog.trace.bootstrap.config.provider; -import java.io.File; +import datadog.cli.CLIHelper; +import datadog.trace.bootstrap.config.provider.stableconfigyaml.ConfigurationMap; +import datadog.trace.bootstrap.config.provider.stableconfigyaml.Rule; +import datadog.trace.bootstrap.config.provider.stableconfigyaml.Selector; +import datadog.trace.bootstrap.config.provider.stableconfigyaml.StableConfigYaml; +import datadog.yaml.YamlParser; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; +import java.util.Collections; import java.util.HashMap; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Stream; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.BiPredicate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class StableConfigParser { private static final Logger log = LoggerFactory.getLogger(StableConfigParser.class); - // Match config_id: - private static final Pattern idPattern = Pattern.compile("^config_id\\s*:(.*)$"); - // Match 'apm_configuration_default:' - private static final Pattern apmConfigPattern = Pattern.compile("^apm_configuration_default:$"); - // Match indented (2 spaces) key-value pairs, either with double quotes or without - private static final Pattern keyValPattern = - Pattern.compile("^\\s{2}([^:]+):\\s*(\"[^\"]*\"|[^\"\\n]*)$");; + private static final Set VM_ARGS = new HashSet<>(CLIHelper.getVmArgs()); + + /** + * Parses a configuration file and returns a stable configuration object. + * + *

This method reads a configuration file from the given file path, parses the YAML content, + * and identifies configurations for the process using a combination of apm_configuration_default + * and apm_configuration_rules. If a matching rule is found in apm_configuration_rules, it returns + * a {@link StableConfigSource.StableConfig} object with the merged configuration. If no matching + * rule is found, it returns the default configuration. If neither a matching rule nor a default + * configuration is found, an empty configuration is returned. + * + * @param filePath The path to the YAML configuration file to be parsed. + * @return A {@link StableConfigSource.StableConfig} object containing the stable configuration. + * @throws IOException If there is an error reading the file or parsing the YAML content. + */ public static StableConfigSource.StableConfig parse(String filePath) throws IOException { - File file = new File(filePath); - if (!file.exists()) { - log.debug("Stable configuration file not available at specified path: {}", file); - return StableConfigSource.StableConfig.EMPTY; + try { + StableConfigYaml data = YamlParser.parse(filePath, StableConfigYaml.class); + + String configId = data.getConfig_id(); + ConfigurationMap configMap = data.getApm_configuration_default(); + List rules = data.getApm_configuration_rules(); + + if (!rules.isEmpty()) { + for (Rule rule : rules) { + // Use the first matching rule + if (doesRuleMatch(rule)) { + // Merge configs found in apm_configuration_rules with those found in + // apm_configuration_default + configMap.putAll(rule.getConfiguration()); + return createStableConfig(configId, configMap); + } + } + } + // If configs were found in apm_configuration_default, use them + if (!configMap.isEmpty()) { + return createStableConfig(configId, configMap); + } + + // If there's a configId but no configMap, use configId but return an empty map + if (configId != null) { + return new StableConfigSource.StableConfig(configId, Collections.emptyMap()); + } + + } catch (IOException e) { + log.debug( + "Stable configuration file either not found or not readable at filepath {}", filePath); + } + return StableConfigSource.StableConfig.EMPTY; + } + + /** + * Checks if the rule's selectors match the current process. All must match for a "true" return + * value. + */ + private static boolean doesRuleMatch(Rule rule) { + for (Selector selector : rule.getSelectors()) { + if (!selectorMatch( + selector.getOrigin(), selector.getMatches(), selector.getOperator(), selector.getKey())) { + return false; // Return false immediately if any selector doesn't match + } + } + return true; // Return true if all selectors match + } + + /** Creates a StableConfig object from the provided configId and configMap. */ + private static StableConfigSource.StableConfig createStableConfig( + String configId, ConfigurationMap configMap) { + return new StableConfigSource.StableConfig(configId, new HashMap<>(configMap)); + } + + private static boolean validOperatorForLanguageOrigin(String operator) { + operator = operator.toLowerCase(); + // "exists" is not valid + switch (operator) { + case "equals": + case "starts_with": + case "ends_with": + case "contains": + return true; + default: + return false; + } + } + + private static boolean checkEnvMatches( + List values, List matches, BiPredicate compareFunc) { + // envValue shouldn't be null, but doing an extra check to avoid NullPointerException on + // compareFunc.test + if (values == null) { + return false; } - Map configMap = new HashMap<>(); - String[] configId = new String[1]; - try (Stream lines = Files.lines(Paths.get(filePath))) { - int apmConfigNotFound = -1, apmConfigStarted = 0, apmConfigComplete = 1; - int[] apmConfigFound = {apmConfigNotFound}; - lines.forEach( - line -> { - Matcher matcher = idPattern.matcher(line); - if (matcher.find()) { - // Do not allow duplicate config_id keys - if (configId[0] != null) { - throw new RuntimeException("Duplicate config_id keys found; file may be malformed"); - } - configId[0] = trimQuotes(matcher.group(1).trim()); - return; // go to next line - } - // TODO: Do not allow duplicate apm_configuration_default keys; and/or return early once - // apmConfigFound[0] == apmConfigComplete - if (apmConfigFound[0] == apmConfigNotFound - && apmConfigPattern.matcher(line).matches()) { - apmConfigFound[0] = apmConfigStarted; - return; // go to next line - } - if (apmConfigFound[0] == apmConfigStarted) { - Matcher keyValueMatcher = keyValPattern.matcher(line); - if (keyValueMatcher.matches()) { - configMap.put( - keyValueMatcher.group(1).trim(), - trimQuotes(keyValueMatcher.group(2).trim())); // Store key-value pair in map - } else { - // If we encounter a non-indented or non-key-value line, stop processing - apmConfigFound[0] = apmConfigComplete; - } - } - }); - return new StableConfigSource.StableConfig(configId[0], configMap); + for (String match : matches) { + if (match == null) { + continue; + } + for (String value : values) { + if (compareFunc.test(value, match.toLowerCase())) { + return true; + } + } } + return false; } - private static String trimQuotes(String value) { - if (value.length() > 1 && (value.startsWith("'") && value.endsWith("'")) - || (value.startsWith("\"") && value.endsWith("\""))) { - return value.substring(1, value.length() - 1); + // We do all of the case insensitivity modifications in this function, because each selector will + // be viewed just once + static boolean selectorMatch(String origin, List matches, String operator, String key) { + switch (origin.toLowerCase()) { + case "language": + if (!validOperatorForLanguageOrigin(operator)) { + return false; + } + for (String entry : matches) { + // loose match on any reference to "*java*" + if (entry.toLowerCase().contains("java")) { + return true; + } + } + case "environment_variables": + if (key == null) { + return false; + } + String envValue = System.getenv(key.toUpperCase()); + if (envValue == null) { + return false; + } + envValue = envValue.toLowerCase(); + switch (operator.toLowerCase()) { + case "exists": + // We don't care about the value + return true; + case "equals": + return checkEnvMatches( + Collections.singletonList(envValue), matches, String::equalsIgnoreCase); + case "starts_with": + return checkEnvMatches( + Collections.singletonList(envValue), matches, String::startsWith); + case "ends_with": + return checkEnvMatches(Collections.singletonList(envValue), matches, String::endsWith); + case "contains": + return checkEnvMatches(Collections.singletonList(envValue), matches, String::contains); + default: + return false; + } + case "process_arguments": + // For now, always return true if `key` exists in the JVM Args + // TODO: flesh out the meaning of each operator for process_arguments + return VM_ARGS.contains(key); + case "tags": + // TODO: Support this down the line (Must define the source of "tags" first) + return false; + default: + return false; } - return value; } } diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/StableConfigSource.java b/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/StableConfigSource.java index df4603e891f..ab634b6fc14 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/StableConfigSource.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/StableConfigSource.java @@ -3,6 +3,7 @@ import static datadog.trace.util.Strings.propertyNameToEnvironmentVariableName; import datadog.trace.api.ConfigOrigin; +import java.io.File; import java.util.Collections; import java.util.Map; import java.util.Set; @@ -23,16 +24,24 @@ public final class StableConfigSource extends ConfigProvider.Source { StableConfigSource.FLEET_STABLE_CONFIG_PATH, ConfigOrigin.FLEET_STABLE_CONFIG); private final ConfigOrigin fileOrigin; - private final StableConfig config; - StableConfigSource(String file, ConfigOrigin origin) { + StableConfigSource(String filePath, ConfigOrigin origin) { this.fileOrigin = origin; + File file = new File(filePath); + if (!file.exists()) { + this.config = StableConfig.EMPTY; + return; + } StableConfig cfg; try { - cfg = StableConfigParser.parse(file); + log.debug("Stable configuration file found at path: {}", file); + cfg = StableConfigParser.parse(filePath); } catch (Throwable e) { - log.debug("Stable configuration file not readable at specified path: {}", file); + log.warn( + "Encountered the following exception when attempting to read stable configuration file at path: {}, dropping configs.", + file, + e); cfg = StableConfig.EMPTY; } this.config = cfg; @@ -61,16 +70,17 @@ public String getConfigId() { public static class StableConfig { public static final StableConfig EMPTY = new StableConfig(null, Collections.emptyMap()); - private final Map apmConfiguration; + private final Map apmConfiguration; private final String configId; - StableConfig(String configId, Map configMap) { + public StableConfig(String configId, Map configMap) { this.configId = configId; this.apmConfiguration = configMap; } public String get(String key) { - return this.apmConfiguration.get(key); + Object value = this.apmConfiguration.get(key); + return (value == null) ? null : String.valueOf(value); } public Set getKeys() { diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/stableconfigyaml/ConfigurationMap.java b/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/stableconfigyaml/ConfigurationMap.java new file mode 100644 index 00000000000..20940733fb2 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/stableconfigyaml/ConfigurationMap.java @@ -0,0 +1,23 @@ +package datadog.trace.bootstrap.config.provider.stableconfigyaml; + +import java.util.HashMap; +import java.util.Map; + +// ConfigurationMap represents configuration key-values found in stable configuration files +public class ConfigurationMap extends HashMap { + public ConfigurationMap() { + super(); + } + + public ConfigurationMap(Map map) { + super(map); + } +} + +class ConfigurationValue { + private final String value; + + public ConfigurationValue(String value) { + this.value = value; + } +} diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/stableconfigyaml/Rule.java b/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/stableconfigyaml/Rule.java new file mode 100644 index 00000000000..dbcca5792e3 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/stableconfigyaml/Rule.java @@ -0,0 +1,38 @@ +package datadog.trace.bootstrap.config.provider.stableconfigyaml; + +import java.util.ArrayList; +import java.util.List; + +// Rule represents a set of selectors and their corresponding configurations found in stable +// configuration files +public class Rule { + private List selectors; + private ConfigurationMap configuration; + + public Rule() { + this.selectors = new ArrayList<>(); + this.configuration = new ConfigurationMap(); + } + + public Rule(List selectors, ConfigurationMap configuration) { + this.selectors = selectors; + this.configuration = configuration; + } + + // Getters and setters + public List getSelectors() { + return selectors; + } + + public void setSelectors(List selectors) { + this.selectors = selectors; + } + + public ConfigurationMap getConfiguration() { + return configuration; + } + + public void setConfiguration(ConfigurationMap configuration) { + this.configuration = configuration; + } +} diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/stableconfigyaml/Selector.java b/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/stableconfigyaml/Selector.java new file mode 100644 index 00000000000..05ce5cb9c6b --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/stableconfigyaml/Selector.java @@ -0,0 +1,58 @@ +package datadog.trace.bootstrap.config.provider.stableconfigyaml; + +import java.util.ArrayList; +import java.util.List; + +public class Selector { + private String origin; + private String key; + private List matches; + private String operator; + + public Selector() { + this.origin = null; + this.key = null; + this.matches = new ArrayList<>(); + this.operator = null; + } + + public Selector(String origin, String key, List matches, String operator) { + this.origin = origin; + this.key = key; + this.matches = matches; + this.operator = operator; + } + + // Getters and setters + public String getOrigin() { + return origin; + } + + public void setOrigin(String origin) { + this.origin = origin; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public List getMatches() { + return matches; + } + + public void setMatches(List matches) { + this.matches = matches; + } + + public String getOperator() { + return operator; + } + + public void setOperator(String operator) { + this.operator = operator; + } +} diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/stableconfigyaml/StableConfigYaml.java b/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/stableconfigyaml/StableConfigYaml.java new file mode 100644 index 00000000000..ec1a2816ed1 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/bootstrap/config/provider/stableconfigyaml/StableConfigYaml.java @@ -0,0 +1,41 @@ +package datadog.trace.bootstrap.config.provider.stableconfigyaml; + +import java.util.ArrayList; +import java.util.List; + +public class StableConfigYaml { + private String config_id; // optional + private ConfigurationMap apm_configuration_default; + private List apm_configuration_rules; // optional + + public StableConfigYaml() { + this.config_id = null; + this.apm_configuration_default = new ConfigurationMap(); + this.apm_configuration_rules = new ArrayList<>(); + } + + // Getters and setters + public String getConfig_id() { + return config_id; + } + + public void setConfig_id(String config_id) { + this.config_id = config_id; + } + + public ConfigurationMap getApm_configuration_default() { + return apm_configuration_default; + } + + public void setApm_configuration_default(ConfigurationMap apm_configuration_default) { + this.apm_configuration_default = apm_configuration_default; + } + + public List getApm_configuration_rules() { + return apm_configuration_rules; + } + + public void setApm_configuration_rules(List apm_configuration_rules) { + this.apm_configuration_rules = apm_configuration_rules; + } +} diff --git a/internal-api/src/test/groovy/datadog/trace/bootstrap/config/provider/StableConfigParserTest.groovy b/internal-api/src/test/groovy/datadog/trace/bootstrap/config/provider/StableConfigParserTest.groovy index f7cc3a0101c..2df560a0eeb 100644 --- a/internal-api/src/test/groovy/datadog/trace/bootstrap/config/provider/StableConfigParserTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/bootstrap/config/provider/StableConfigParserTest.groovy @@ -6,29 +6,40 @@ import java.nio.file.Files import java.nio.file.Path class StableConfigParserTest extends DDSpecification { - - def "test parser"() { + def "test parse valid"() { when: Path filePath = StableConfigSourceTest.tempFile() if (filePath == null) { throw new AssertionError("Failed to create test file") } + injectEnvConfig("DD_SERVICE", "mysvc") + // From the below yaml, only apm_configuration_default and the second selector should be applied: We use the first matching rule and discard the rest String yaml = """ -something-irrelevant: "" config_id: 12345 -something : not : expected << and weird format - inufjka << - [a, - b, - c, - d] apm_configuration_default: - KEY_ONE: value_one - KEY_TWO: "value_two" - KEY_THREE: 100 - KEY_FOUR: true - KEY_FIVE: [a,b,c,d] -something-else-irrelevant: value-irrelevant + KEY_ONE: "default" + KEY_TWO: true +apm_configuration_rules: + - selectors: + - origin: language + matches: ["golang"] + operator: equals + configuration: + KEY_ONE: "ignored" + - selectors: + - origin: language + matches: ["Java"] + operator: equals + configuration: + KEY_ONE: "rules" + KEY_THREE: 1 + - selectors: + - origin: environment_variables + key: "DD_SERVICE" + operator: equals + matches: ["mysvc"] + configuration: + KEY_FOUR: "ignored" """ try { StableConfigSourceTest.writeFileRaw(filePath, yaml) @@ -45,31 +56,64 @@ something-else-irrelevant: value-irrelevant then: def keys = cfg.getKeys() - keys.size() == 5 - !keys.contains("something-irrelevant") - !keys.contains("something-else-irrelevant") + keys.size() == 3 cfg.getConfigId().trim() == ("12345") - cfg.get("KEY_ONE") == "value_one" - cfg.get("KEY_TWO") == "value_two" - cfg.get("KEY_THREE") == "100" - cfg.get("KEY_FOUR") == "true" - cfg.get("KEY_FIVE") == "[a,b,c,d]" + cfg.get("KEY_ONE") == "rules" + cfg.get("KEY_TWO") == "true" + cfg.get("KEY_THREE") == "1" Files.delete(filePath) } - def "test duplicate config_id"() { + def "test selectorMatch"() { + when: + // Env vars + injectEnvConfig("DD_PROFILING_ENABLED", "true") + injectEnvConfig("DD_SERVICE", "mysvc") + injectEnvConfig("DD_TAGS", "team:apm,component:web") + def match = StableConfigParser.selectorMatch(origin, matches, operator, key) + + then: + match == expectMatch + + where: + origin | matches | operator | key | expectMatch + "language" | ["java"] | "equals" | "" | true + "LANGUAGE" | ["JaVa"] | "EQUALS" | "" | true // check case insensitivity + "language" | ["java", "golang"] | "equals" | "" | true + "language" | ["java"] | "starts_with" | "" | true + "language" | ["golang"] | "equals" | "" | false + "environment_variables" | [] | "exists" | "DD_TAGS" | true + "environment_variables" | ["team:apm"] | "contains" | "DD_TAGS" | true + "ENVIRONMENT_VARIABLES" | ["TeAm:ApM"] | "CoNtAiNs" | "Dd_TaGs" | true // check case insensitivity + "environment_variables" | ["team:apm"] | "equals" | "DD_TAGS" | false + "environment_variables" | ["team:apm"] | "starts_with" | "DD_TAGS" | true + "environment_variables" | ["true"] | "equals" | "DD_PROFILING_ENABLED" | true + "environment_variables" | ["abcdefg"] | "equals" | "DD_PROFILING_ENABLED" | false + "environment_variables" | ["true"] | "equals" | "DD_PROFILING_ENABLED" | true + "environment_variables" | ["mysvc", "othersvc"] | "equals" | "DD_SERVICE" | true + "environment_variables" | ["my"] | "starts_with" | "DD_SERVICE" | true + "environment_variables" | ["svc"] | "ends_with" | "DD_SERVICE" | true + "environment_variables" | ["svc"] | "contains" | "DD_SERVICE" | true + "environment_variables" | ["other"] | "contains" | "DD_SERVICE" | false + "environment_variables" | [null] | "contains" | "DD_SERVICE" | false + // "process_arguments" | null | "equals" | "-DCustomKey" | true + } + + def "test duplicate entries"() { + // When duplicate keys are encountered, snakeyaml preserves the last value by default when: Path filePath = StableConfigSourceTest.tempFile() if (filePath == null) { throw new AssertionError("Failed to create test file") } String yaml = """ -config_id: 12345 -something-irrelevant: "" -apm_configuration_default: - DD_KEY: value -config_id: 67890 -""" + config_id: 12345 + config_id: 67890 + apm_configuration_default: + DD_KEY: value_1 + apm_configuration_default: + DD_KEY: value_2 + """ try { StableConfigSourceTest.writeFileRaw(filePath, yaml) @@ -77,34 +121,43 @@ config_id: 67890 throw new AssertionError("Failed to write to file: ${e.message}") } - Exception exception StableConfigSource.StableConfig cfg try { cfg = StableConfigParser.parse(filePath.toString()) } catch (Exception e) { - exception = e + throw new AssertionError("Failed to parse the file: ${e.message}") } then: - cfg == null - exception != null - exception.getMessage() == "Duplicate config_id keys found; file may be malformed" + cfg != null + cfg.getConfigId() == "67890" + cfg.get("DD_KEY") == "value_2" } - def "test duplicate apm_configuration_default"() { - // Assert that only the first entry is used + def "test parse invalid"() { + // If any piece of the file is invalid, the whole file is rendered invalid and an exception is thrown when: Path filePath = StableConfigSourceTest.tempFile() if (filePath == null) { throw new AssertionError("Failed to create test file") } String yaml = """ -apm_configuration_default: - KEY_1: value_1 -something-else-irrelevant: value-irrelevant -apm_configuration_default: - KEY_2: value_2 -""" + something-irrelevant: "" + config_id: 12345 + something : not : expected << and weird format + inufjka << + [a, + b, + c, + d] + apm_configuration_default: + KEY_ONE: value_one + KEY_TWO: "value_two" + KEY_THREE: 100 + KEY_FOUR: true + KEY_FIVE: [a,b,c,d] + something-else-irrelevant: value-irrelevant + """ try { StableConfigSourceTest.writeFileRaw(filePath, yaml) } catch (IOException e) { @@ -112,16 +165,16 @@ apm_configuration_default: } StableConfigSource.StableConfig cfg + Exception exception = null try { cfg = StableConfigParser.parse(filePath.toString()) } catch (Exception e) { - throw new AssertionError("Failed to parse the file: ${e.message}") + exception = e } then: - def keys = cfg.getKeys() - keys.size() == 1 - !keys.contains("KEY_2") - cfg.get("KEY_1") == "value_1" + exception != null + cfg == null + Files.delete(filePath) } } diff --git a/internal-api/src/test/groovy/datadog/trace/bootstrap/config/provider/StableConfigSourceTest.groovy b/internal-api/src/test/groovy/datadog/trace/bootstrap/config/provider/StableConfigSourceTest.groovy index 890ab15a7b9..b6596174985 100644 --- a/internal-api/src/test/groovy/datadog/trace/bootstrap/config/provider/StableConfigSourceTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/bootstrap/config/provider/StableConfigSourceTest.groovy @@ -1,13 +1,23 @@ package datadog.trace.bootstrap.config.provider +import static java.util.Collections.singletonMap + import datadog.trace.api.ConfigOrigin +import datadog.trace.bootstrap.config.provider.stableconfigyaml.ConfigurationMap +import datadog.trace.bootstrap.config.provider.stableconfigyaml.ConfigurationValue +import datadog.trace.bootstrap.config.provider.stableconfigyaml.Rule +import datadog.trace.bootstrap.config.provider.stableconfigyaml.Selector +import datadog.trace.bootstrap.config.provider.stableconfigyaml.StableConfigYaml import datadog.trace.test.util.DDSpecification import org.yaml.snakeyaml.DumperOptions import org.yaml.snakeyaml.Yaml -import spock.lang.Shared +import org.yaml.snakeyaml.introspector.Property +import org.yaml.snakeyaml.nodes.NodeTuple +import org.yaml.snakeyaml.nodes.Tag +import org.yaml.snakeyaml.representer.Representer -import java.nio.file.Path import java.nio.file.Files +import java.nio.file.Path import java.nio.file.StandardOpenOption class StableConfigSourceTest extends DDSpecification { @@ -34,34 +44,8 @@ class StableConfigSourceTest extends DDSpecification { config.getConfigId() == null } - def "test get"() { - when: - Path filePath = tempFile() - if (filePath == null) { - throw new AssertionError("Failed to create test file") - } - - def configs = new HashMap<>() << ["DD_SERVICE": "svc", "DD_ENV": "env", "CONFIG_NO_DD": "value123"] - - try { - writeFileYaml(filePath, "12345", configs) - } catch (IOException e) { - throw new AssertionError("Failed to write to file: ${e.message}") - } - - StableConfigSource cfg = new StableConfigSource(filePath.toString(), ConfigOrigin.LOCAL_STABLE_CONFIG) - - then: - cfg.get("service") == "svc" - cfg.get("env") == "env" - cfg.get("config_no_dd") == null - cfg.get("config_nonexistent") == null - cfg.getKeys().size() == 3 - cfg.getConfigId() == "12345" - Files.delete(filePath) - } - def "test file invalid format"() { + // StableConfigSource must handle the exception thrown by StableConfigParser.parse(filePath) gracefully when: Path filePath = tempFile() if (filePath == null) { @@ -69,7 +53,7 @@ class StableConfigSourceTest extends DDSpecification { } try { - writeFileRaw(filePath, configId, configs) + writeFileRaw(filePath, configId, data) } catch (IOException e) { throw new AssertionError("Failed to write to file: ${e.message}") } @@ -82,7 +66,7 @@ class StableConfigSourceTest extends DDSpecification { Files.delete(filePath) where: - configId | configs + configId | data null | corruptYaml "12345" | "this is not yaml format!" } @@ -93,9 +77,12 @@ class StableConfigSourceTest extends DDSpecification { if (filePath == null) { throw new AssertionError("Failed to create test file") } + StableConfigYaml stableConfigYaml = new StableConfigYaml() + stableConfigYaml.setConfig_id(configId) + stableConfigYaml.setApm_configuration_default(defaultConfigs as ConfigurationMap) try { - writeFileYaml(filePath, configId, configs) + writeFileYaml(filePath, stableConfigYaml) } catch (IOException e) { throw new AssertionError("Failed to write to file: ${e.message}") } @@ -103,28 +90,49 @@ class StableConfigSourceTest extends DDSpecification { StableConfigSource stableCfg = new StableConfigSource(filePath.toString(), ConfigOrigin.LOCAL_STABLE_CONFIG) then: - for (key in configs.keySet()) { + for (key in defaultConfigs.keySet()) { String keyString = (String) key keyString = keyString.substring(4) // Cut `DD_` - stableCfg.get(keyString) == configs.get(key) + stableCfg.get(keyString) == defaultConfigs.get(key) + } + // All configs from MatchingRule should be applied + if (ruleConfigs.contains(sampleMatchingRule)) { + for (key in sampleMatchingRule.getConfiguration().keySet()) { + String keyString = (String) key + keyString = keyString.substring(4) // Cut `DD_` + stableCfg.get(keyString) == defaultConfigs.get(key) + } + } + // None of the configs from NonMatchingRule should be applied + if (ruleConfigs.contains(sampleNonMatchingRule)) { + Set cfgKeys = stableCfg.getKeys() + for (key in sampleMatchingRule.getConfiguration().keySet()) { + String keyString = (String) key + keyString = keyString.substring(4) // Cut `DD_` + !cfgKeys.contains(keyString) + } } Files.delete(filePath) where: - configId | configs - "" | new HashMap<>() - "12345" | new HashMap<>() << ["DD_KEY_ONE": "one", "DD_KEY_TWO": "two"] + configId | defaultConfigs | ruleConfigs + "" | new HashMap<>() | Arrays.asList(new Rule()) + "12345" | new HashMap<>() << ["DD_KEY_ONE": "one", "DD_KEY_TWO": "two"] | Arrays.asList(sampleMatchingRule, sampleNonMatchingRule) } // Corrupt YAML string variable used for testing, defined outside the 'where' block for readability - @Shared - def corruptYaml = ''' + def static corruptYaml = ''' abc: 123 def: ghi: "jkl" lmn: 456 ''' + // Matching and non-matching Rules used for testing, defined outside the 'where' block for readability + def static sampleMatchingRule = new Rule(Arrays.asList(new Selector("origin", "language", Arrays.asList("Java"), null)), new ConfigurationMap(singletonMap("DD_KEY_THREE", new ConfigurationValue("three")))) + def static sampleNonMatchingRule = new Rule(Arrays.asList(new Selector("origin", "language", Arrays.asList("Golang"), null)), new ConfigurationMap(singletonMap("DD_KEY_FOUR", new ConfigurationValue("four")))) + + // Helper functions static Path tempFile() { try { return Files.createTempFile("testFile_", ".yaml") @@ -135,34 +143,45 @@ class StableConfigSourceTest extends DDSpecification { } } - static writeFileYaml(Path filePath, String configId, Map configs) { - DumperOptions options = new DumperOptions() - options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK) - - // Prepare to write the data map to the file in yaml format - Yaml yaml = new Yaml(options) - String yamlString - Map data = new HashMap<>() - if (configId != null) { - data.put("config_id", configId) - } - if (configs instanceof HashMap) { - data.put("apm_configuration_default", configs) - } + def stableConfigYamlWriter = getStableConfigYamlWriter() - yamlString = yaml.dump(data) + Yaml getStableConfigYamlWriter() { + DumperOptions options = new DumperOptions() + // Create the Representer, configure it to omit nulls + Representer representer = new Representer(options) { + @Override + protected NodeTuple representJavaBeanProperty(Object javaBean, Property property, Object propertyValue, Tag customTag) { + if (propertyValue == null) { + return null // Skip null values + } else { + return super.representJavaBeanProperty(javaBean, property, propertyValue, customTag) + } + } + } + // Exclude class tag from the resulting yaml string + representer.addClassTag(StableConfigYaml.class, Tag.MAP) + + // YAML instance with custom Representer + return new Yaml(representer, options) + } - StandardOpenOption[] openOpts = [StandardOpenOption.WRITE] as StandardOpenOption[] - Files.write(filePath, yamlString.getBytes(), openOpts) + def writeFileYaml(Path filePath, StableConfigYaml stableConfigs) { + // Yaml yaml = getStableConfigYaml(); + try (FileWriter writer = new FileWriter(filePath.toString())) { + stableConfigYamlWriter.dump(stableConfigs, writer) + } catch (IOException e) { + e.printStackTrace() + } } // Use this if you want to explicitly write/test configId - def writeFileRaw(Path filePath, String configId, String configs) { - String data = configId + "\n" + configs + def writeFileRaw(Path filePath, String configId, String data) { + data = configId + "\n" + data StandardOpenOption[] openOpts = [StandardOpenOption.WRITE] as StandardOpenOption[] Files.write(filePath, data.getBytes(), openOpts) } + // Use this for writing a string directly into a file static writeFileRaw(Path filePath, String data) { StandardOpenOption[] openOpts = [StandardOpenOption.WRITE] as StandardOpenOption[] Files.write(filePath, data.getBytes(), openOpts) diff --git a/settings.gradle b/settings.gradle index 5cf6d37e8f8..299c277d138 100644 --- a/settings.gradle +++ b/settings.gradle @@ -89,6 +89,7 @@ include ':communication' include ':components:cli' include ':components:context' include ':components:json' +include ':components:yaml' include ':telemetry' include ':remote-config:remote-config-api' include ':remote-config:remote-config-core'