From bd68acca1956c5cc2d75dacac0d560c1ddc2e06c Mon Sep 17 00:00:00 2001 From: Ben Donnelly Date: Wed, 6 Dec 2023 14:16:23 +0000 Subject: [PATCH 1/4] feat(metrics): initial work for metric tracepoints --- .github/workflows/on_release.yml | 12 ++ .idea/encodings.xml | 4 + .sdkman | 1 - .sdkmanrc | 1 + .../com/intergral/deep/agent/api/IDeep.java | 4 +- .../agent/api/plugin/IMetricProcessor.java | 31 ++++ .../agent/api/plugin/MetricDefinition.java | 70 +++++++++ .../agent/api/spi/ConditionalProvider.java | 22 +++ .../deep/agent/api/spi/MetricProvider.java | 24 +++ agent/pom.xml | 6 + .../com/intergral/deep/agent/DeepAgent.java | 7 +- .../deep/agent/poll/LongPollService.java | 30 +++- .../deep/agent/resource/SpiUtil.java | 14 +- .../deep/agent/settings/Settings.java | 12 ++ .../tracepoint/TracepointConfigService.java | 7 +- .../tracepoint/handler/FrameCollector.java | 33 +++++ .../tracepoint/handler/FrameProcessor.java | 60 +++++++- .../inst/jsp/JSPMappedBreakpoint.java | 2 +- .../deep/agent/types/TracePointConfig.java | 30 +++- ...ntergral.deep.agent.api.spi.MetricProvider | 18 +++ .../TracepointConfigServiceTest.java | 16 +- .../deep/test/MockEventSnapshot.java | 2 +- .../deep/test/MockTracepointConfig.java | 6 +- deep/pom.xml | 22 +++ .../com/intergral/deep/examples/Main.java | 2 +- .../com/intergral/deep/examples/Main.java | 4 +- examples/pom.xml | 1 + examples/prometheus-metrics-example/pom.xml | 70 +++++++++ .../com/intergral/deep/examples/BaseTest.java | 50 +++++++ .../com/intergral/deep/examples/Main.java | 135 +++++++++++++++++ .../intergral/deep/examples/SimpleTest.java | 88 +++++++++++ plugins/pom.xml | 1 + plugins/prometheus-metrics/pom.xml | 50 +++++++ .../deep/plugin/PrometheusMetricsPlugin.java | 138 ++++++++++++++++++ pom.xml | 3 +- 35 files changed, 941 insertions(+), 35 deletions(-) delete mode 100644 .sdkman create mode 100644 agent-api/src/main/java/com/intergral/deep/agent/api/plugin/IMetricProcessor.java create mode 100644 agent-api/src/main/java/com/intergral/deep/agent/api/plugin/MetricDefinition.java create mode 100644 agent-api/src/main/java/com/intergral/deep/agent/api/spi/ConditionalProvider.java create mode 100644 agent-api/src/main/java/com/intergral/deep/agent/api/spi/MetricProvider.java create mode 100644 agent/src/main/resources/META-INF/services/com.intergral.deep.agent.api.spi.MetricProvider create mode 100644 examples/prometheus-metrics-example/pom.xml create mode 100644 examples/prometheus-metrics-example/src/main/java/com/intergral/deep/examples/BaseTest.java create mode 100644 examples/prometheus-metrics-example/src/main/java/com/intergral/deep/examples/Main.java create mode 100644 examples/prometheus-metrics-example/src/main/java/com/intergral/deep/examples/SimpleTest.java create mode 100644 plugins/prometheus-metrics/pom.xml create mode 100644 plugins/prometheus-metrics/src/main/java/com/intergral/deep/plugin/PrometheusMetricsPlugin.java diff --git a/.github/workflows/on_release.yml b/.github/workflows/on_release.yml index 0da1d5d..1622164 100644 --- a/.github/workflows/on_release.yml +++ b/.github/workflows/on_release.yml @@ -33,6 +33,12 @@ jobs: - name: Build and Release run: mvn -s .ci-settings.xml clean deploy -DskipTests -P release-ossrh -B -U -pl agent,deep --also-make + - name: Get release + id: get_release + uses: bruceadams/get-release@v1.3.2 + env: + GITHUB_TOKEN: ${{ github.token }} + - name: Upload release binary uses: actions/upload-release-asset@v1.0.2 env: @@ -64,6 +70,12 @@ jobs: - name: Zip Site run: tar -czf site.tar.gz target/staging + - name: Get release + id: get_release + uses: bruceadams/get-release@v1.3.2 + env: + GITHUB_TOKEN: ${{ github.token }} + - uses: actions/upload-release-asset@v1.0.2 env: GITHUB_TOKEN: ${{ github.token }} diff --git a/.idea/encodings.xml b/.idea/encodings.xml index ab43bce..8f5c7ff 100644 --- a/.idea/encodings.xml +++ b/.idea/encodings.xml @@ -13,6 +13,8 @@ + + @@ -25,6 +27,8 @@ + + diff --git a/.sdkman b/.sdkman deleted file mode 100644 index 09adfc0..0000000 --- a/.sdkman +++ /dev/null @@ -1 +0,0 @@ -use java 13.0.2-open \ No newline at end of file diff --git a/.sdkmanrc b/.sdkmanrc index 1d93138..18fd0a5 100644 --- a/.sdkmanrc +++ b/.sdkmanrc @@ -1,3 +1,4 @@ # Enable auto-env through the sdkman_auto_env config # Add key=value pairs of SDKs to use below java=13.0.2-open +maven=3.9.6 diff --git a/agent-api/src/main/java/com/intergral/deep/agent/api/IDeep.java b/agent-api/src/main/java/com/intergral/deep/agent/api/IDeep.java index 2148787..d549c97 100644 --- a/agent-api/src/main/java/com/intergral/deep/agent/api/IDeep.java +++ b/agent-api/src/main/java/com/intergral/deep/agent/api/IDeep.java @@ -19,6 +19,7 @@ import com.intergral.deep.agent.api.plugin.IPlugin; import com.intergral.deep.agent.api.plugin.IPlugin.IPluginRegistration; +import com.intergral.deep.agent.api.plugin.MetricDefinition; import com.intergral.deep.agent.api.tracepoint.ITracepoint.ITracepointRegistration; import java.util.Collection; import java.util.Map; @@ -80,8 +81,9 @@ public interface IDeep { * @param line the line number * @param args the key value pairs that further define the tracepoint * @param watches the list of watch expressions + * @param metrics the list of metric expressions * @return a {@link ITracepointRegistration} that can be used to remove the tracepoint */ ITracepointRegistration registerTracepoint(final String path, final int line, final Map args, - final Collection watches); + final Collection watches, final Collection metrics); } diff --git a/agent-api/src/main/java/com/intergral/deep/agent/api/plugin/IMetricProcessor.java b/agent-api/src/main/java/com/intergral/deep/agent/api/plugin/IMetricProcessor.java new file mode 100644 index 0000000..88bdf1a --- /dev/null +++ b/agent-api/src/main/java/com/intergral/deep/agent/api/plugin/IMetricProcessor.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2023 Intergral GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.intergral.deep.agent.api.plugin; + +import java.util.Map; + +public interface IMetricProcessor { + + void counter(final String name, final Map tags, final String namespace, final String help, final Double value); + + void gauge(final String name, final Map tags, final String namespace, final String help, final Double value); + + void histogram(final String name, final Map tags, final String namespace, final String help, final Double value); + + void summary(final String name, final Map tags, final String namespace, final String help, final Double value); +} diff --git a/agent-api/src/main/java/com/intergral/deep/agent/api/plugin/MetricDefinition.java b/agent-api/src/main/java/com/intergral/deep/agent/api/plugin/MetricDefinition.java new file mode 100644 index 0000000..7d3e629 --- /dev/null +++ b/agent-api/src/main/java/com/intergral/deep/agent/api/plugin/MetricDefinition.java @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2023 Intergral GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.intergral.deep.agent.api.plugin; + +import java.util.Map; + +public class MetricDefinition { + + private final String name; + private final Map tags; + private final String type; + private final String expression; + private final String namespace; + private final String help; + + public MetricDefinition( + final String name, + final Map tags, + final String type, + final String expression, + final String namespace, + final String help) { + + this.name = name; + this.tags = tags; + this.type = type; + this.expression = expression; + this.namespace = namespace; + this.help = help; + } + + public String getName() { + return name; + } + + public Map getTags() { + return tags; + } + + public String getType() { + return type; + } + + public String getExpression() { + return expression; + } + + public String getNamespace() { + return namespace; + } + + public String getHelp() { + return help; + } +} diff --git a/agent-api/src/main/java/com/intergral/deep/agent/api/spi/ConditionalProvider.java b/agent-api/src/main/java/com/intergral/deep/agent/api/spi/ConditionalProvider.java new file mode 100644 index 0000000..7936eb3 --- /dev/null +++ b/agent-api/src/main/java/com/intergral/deep/agent/api/spi/ConditionalProvider.java @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2023 Intergral GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.intergral.deep.agent.api.spi; +public interface ConditionalProvider { + + boolean isActive(); +} diff --git a/agent-api/src/main/java/com/intergral/deep/agent/api/spi/MetricProvider.java b/agent-api/src/main/java/com/intergral/deep/agent/api/spi/MetricProvider.java new file mode 100644 index 0000000..06efc39 --- /dev/null +++ b/agent-api/src/main/java/com/intergral/deep/agent/api/spi/MetricProvider.java @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2023 Intergral GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.intergral.deep.agent.api.spi; + +import com.intergral.deep.agent.api.plugin.IMetricProcessor; + +public interface MetricProvider extends IMetricProcessor, Ordered { + +} diff --git a/agent/pom.xml b/agent/pom.xml index b29fc47..2a1e730 100644 --- a/agent/pom.xml +++ b/agent/pom.xml @@ -77,6 +77,12 @@ 1.0-SNAPSHOT compile + + com.intergral.deep.plugins + prometheus-metrics + 1.0-SNAPSHOT + provided + diff --git a/agent/src/main/java/com/intergral/deep/agent/DeepAgent.java b/agent/src/main/java/com/intergral/deep/agent/DeepAgent.java index b04f400..a6b72df 100644 --- a/agent/src/main/java/com/intergral/deep/agent/DeepAgent.java +++ b/agent/src/main/java/com/intergral/deep/agent/DeepAgent.java @@ -23,6 +23,7 @@ import com.intergral.deep.agent.api.logger.ITracepointLogger; import com.intergral.deep.agent.api.plugin.IPlugin; import com.intergral.deep.agent.api.plugin.IPlugin.IPluginRegistration; +import com.intergral.deep.agent.api.plugin.MetricDefinition; import com.intergral.deep.agent.api.resource.Resource; import com.intergral.deep.agent.api.settings.ISettings; import com.intergral.deep.agent.api.spi.ResourceProvider; @@ -156,13 +157,13 @@ public IPlugin get() { @Override public ITracepointRegistration registerTracepoint(final String path, final int line) { - return registerTracepoint(path, line, Collections.emptyMap(), Collections.emptyList()); + return registerTracepoint(path, line, Collections.emptyMap(), Collections.emptyList(), Collections.emptyList()); } @Override public ITracepointRegistration registerTracepoint(final String path, final int line, final Map args, - final Collection watches) { - final TracePointConfig tracePointConfig = this.tracepointConfig.addCustom(path, line, args, watches); + final Collection watches, final Collection metricDefinitions) { + final TracePointConfig tracePointConfig = this.tracepointConfig.addCustom(path, line, args, watches, metricDefinitions); return new ITracepointRegistration() { @Override public void unregister() { diff --git a/agent/src/main/java/com/intergral/deep/agent/poll/LongPollService.java b/agent/src/main/java/com/intergral/deep/agent/poll/LongPollService.java index 1a23e2c..70c8a33 100644 --- a/agent/src/main/java/com/intergral/deep/agent/poll/LongPollService.java +++ b/agent/src/main/java/com/intergral/deep/agent/poll/LongPollService.java @@ -17,6 +17,7 @@ package com.intergral.deep.agent.poll; +import com.intergral.deep.agent.api.plugin.MetricDefinition; import com.intergral.deep.agent.api.resource.Resource; import com.intergral.deep.agent.grpc.GrpcService; import com.intergral.deep.agent.settings.Settings; @@ -28,6 +29,7 @@ import com.intergral.deep.proto.poll.v1.PollRequest; import com.intergral.deep.proto.poll.v1.PollResponse; import com.intergral.deep.proto.poll.v1.ResponseType; +import com.intergral.deep.proto.tracepoint.v1.Metric; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -38,6 +40,7 @@ * This service deals with polling the remote service for tracepoint configs. */ public class LongPollService implements ITimerTask { + private final Settings settings; private final GrpcService grpcService; private final DriftAwareThread thread; @@ -106,10 +109,35 @@ private Collection convertResponse( tracePointConfig.getPath(), tracePointConfig.getLineNumber(), Collections.unmodifiableMap(new HashMap<>(tracePointConfig.getArgsMap())), - Collections.unmodifiableCollection(tracePointConfig.getWatchesList()))) + Collections.unmodifiableCollection(tracePointConfig.getWatchesList()), + Collections.unmodifiableList(covertMetrics(tracePointConfig.getMetricsList())))) + .collect(Collectors.toList()); + } + + private List covertMetrics(final List metricsList) { + if (metricsList == null || metricsList.isEmpty()) { + return Collections.emptyList(); + } + return metricsList.stream().map( + metric -> new MetricDefinition(metric.getName(), metric.getTagsMap(), metric.getType().toString(), metric.getExpression(), + namespaceOrDefault(metric.getNamespace()), helpOrDefault(metric.getHelp(), metric.getExpression()))) .collect(Collectors.toList()); } + private String namespaceOrDefault(final String namespace) { + if (namespace == null || namespace.trim().isEmpty()) { + return "deep_metrics"; + } + return namespace; + } + + private String helpOrDefault(final String help, final String expression) { + if (help == null || help.trim().isEmpty()) { + return "Metric generated from expression: " + expression; + } + return help; + } + private com.intergral.deep.proto.resource.v1.Resource buildResource() { final Resource resource = this.settings.getResource(); return com.intergral.deep.proto.resource.v1.Resource.newBuilder() diff --git a/agent/src/main/java/com/intergral/deep/agent/resource/SpiUtil.java b/agent/src/main/java/com/intergral/deep/agent/resource/SpiUtil.java index fe2bee7..df0b527 100644 --- a/agent/src/main/java/com/intergral/deep/agent/resource/SpiUtil.java +++ b/agent/src/main/java/com/intergral/deep/agent/resource/SpiUtil.java @@ -5,11 +5,13 @@ package com.intergral.deep.agent.resource; +import com.intergral.deep.agent.api.spi.ConditionalProvider; import com.intergral.deep.agent.api.spi.Ordered; import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.ServiceLoader; +import java.util.stream.Collectors; /** * Utilities to load SPI services. @@ -19,11 +21,21 @@ public final class SpiUtil { private SpiUtil() { } - static List loadOrdered(Class spiClass, + public static List loadOrdered(Class spiClass, ClassLoader serviceClassLoader) { return loadOrdered(spiClass, serviceClassLoader, ServiceLoader::load); } + public static List loadConditional(Class spiClass, ClassLoader serviceClassLoader) { + final List ordered = loadOrdered(spiClass, serviceClassLoader, ServiceLoader::load); + return ordered.stream().filter(s -> { + if (s instanceof ConditionalProvider) { + return ((ConditionalProvider) s).isActive(); + } + return true; + }).collect(Collectors.toList()); + } + // VisibleForTesting static List loadOrdered( Class spiClass, ClassLoader serviceClassLoader, ServiceLoaderFinder serviceLoaderFinder) { diff --git a/agent/src/main/java/com/intergral/deep/agent/settings/Settings.java b/agent/src/main/java/com/intergral/deep/agent/settings/Settings.java index 67043e2..9a7adac 100644 --- a/agent/src/main/java/com/intergral/deep/agent/settings/Settings.java +++ b/agent/src/main/java/com/intergral/deep/agent/settings/Settings.java @@ -19,9 +19,12 @@ import com.intergral.deep.agent.api.logger.ITracepointLogger; import com.intergral.deep.agent.api.logger.TracepointLogger; +import com.intergral.deep.agent.api.plugin.IMetricProcessor; import com.intergral.deep.agent.api.plugin.IPlugin; import com.intergral.deep.agent.api.resource.Resource; import com.intergral.deep.agent.api.settings.ISettings; +import com.intergral.deep.agent.api.spi.MetricProvider; +import com.intergral.deep.agent.resource.SpiUtil; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; @@ -51,6 +54,7 @@ public class Settings implements ISettings { private Resource resource; private Collection plugins = Collections.emptyList(); private ITracepointLogger tracepointLogger = new TracepointLogger(); + private IMetricProcessor metricProcessor; private Settings(Properties properties) { this.properties = properties; @@ -415,6 +419,14 @@ public void setTracepointLogger(final ITracepointLogger tracepointLogger) { this.tracepointLogger = tracepointLogger; } + public IMetricProcessor getMetricProcessor() { + final List metricProviders = SpiUtil.loadOrdered(MetricProvider.class, Thread.currentThread().getContextClassLoader()); + if(metricProviders.isEmpty()){ + return null; + } + return metricProviders.get(0); + } + /** * Used to indicate an invalid config value. */ diff --git a/agent/src/main/java/com/intergral/deep/agent/tracepoint/TracepointConfigService.java b/agent/src/main/java/com/intergral/deep/agent/tracepoint/TracepointConfigService.java index 70ac34b..c62d3c0 100644 --- a/agent/src/main/java/com/intergral/deep/agent/tracepoint/TracepointConfigService.java +++ b/agent/src/main/java/com/intergral/deep/agent/tracepoint/TracepointConfigService.java @@ -17,6 +17,7 @@ package com.intergral.deep.agent.tracepoint; +import com.intergral.deep.agent.api.plugin.MetricDefinition; import com.intergral.deep.agent.tracepoint.inst.TracepointInstrumentationService; import com.intergral.deep.agent.types.TracePointConfig; import java.util.ArrayList; @@ -86,12 +87,14 @@ public Collection loadTracepointConfigs(final Collection args, - final Collection watches) { - final TracePointConfig tracePointConfig = new TracePointConfig(UUID.randomUUID().toString(), path, line, args, watches); + final Collection watches, final Collection metrics) { + final TracePointConfig tracePointConfig = new TracePointConfig(UUID.randomUUID().toString(), path, line, args, watches, + metrics); this.customTracepoints.add(tracePointConfig); this.processChange(); return tracePointConfig; diff --git a/agent/src/main/java/com/intergral/deep/agent/tracepoint/handler/FrameCollector.java b/agent/src/main/java/com/intergral/deep/agent/tracepoint/handler/FrameCollector.java index f0d8c90..7e396d5 100644 --- a/agent/src/main/java/com/intergral/deep/agent/tracepoint/handler/FrameCollector.java +++ b/agent/src/main/java/com/intergral/deep/agent/tracepoint/handler/FrameCollector.java @@ -379,6 +379,25 @@ public Map variables() { public String logString() { return Utils.valueOf(result); } + + @Override + public Number numberValue() { + if (result instanceof Number) { + // todo should we wrap this is new Number() ? + return (Number) result; + } + final String string = Utils.valueOf(result); + try { + return Double.valueOf(string); + } catch (NumberFormatException nfe) { + return Double.NaN; + } + } + + @Override + public boolean isError() { + return false; + } }; } catch (Throwable t) { return new IExpressionResult() { @@ -397,6 +416,16 @@ public Map variables() { public String logString() { return Utils.throwableToString(t); } + + @Override + public Number numberValue() { + return Double.NaN; + } + + @Override + public boolean isError() { + return true; + } }; } } @@ -515,6 +544,10 @@ protected interface IExpressionResult { Map variables(); String logString(); + + Number numberValue(); + + boolean isError(); } /** diff --git a/agent/src/main/java/com/intergral/deep/agent/tracepoint/handler/FrameProcessor.java b/agent/src/main/java/com/intergral/deep/agent/tracepoint/handler/FrameProcessor.java index b2720e5..924db85 100644 --- a/agent/src/main/java/com/intergral/deep/agent/tracepoint/handler/FrameProcessor.java +++ b/agent/src/main/java/com/intergral/deep/agent/tracepoint/handler/FrameProcessor.java @@ -21,7 +21,9 @@ import com.intergral.deep.agent.Utils; import com.intergral.deep.agent.api.plugin.EvaluationException; import com.intergral.deep.agent.api.plugin.IEvaluator; +import com.intergral.deep.agent.api.plugin.IMetricProcessor; import com.intergral.deep.agent.api.plugin.ISnapshotContext; +import com.intergral.deep.agent.api.plugin.MetricDefinition; import com.intergral.deep.agent.api.reflection.IReflection; import com.intergral.deep.agent.api.resource.Resource; import com.intergral.deep.agent.settings.Settings; @@ -30,7 +32,9 @@ import com.intergral.deep.agent.types.snapshot.WatchResult; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.Map; +import java.util.Map.Entry; import java.util.stream.Collectors; /** @@ -124,7 +128,7 @@ public void configureSelf() { public Collection collect() { final Collection snapshots = new ArrayList<>(); - final IFrameResult processedFrame = super.processFrame(); + final IFrameResult processedFrame = processFrame(); for (final TracePointConfig tracepoint : filteredTracepoints) { try { @@ -135,7 +139,7 @@ public Collection collect() { processedFrame.variables()); for (String watch : tracepoint.getWatches()) { - final FrameCollector.IExpressionResult result = super.evaluateWatchExpression(watch); + final FrameCollector.IExpressionResult result = evaluateWatchExpression(watch); snapshot.addWatchResult(result.result()); snapshot.mergeVariables(result.variables()); } @@ -151,7 +155,14 @@ public Collection collect() { this.logTracepoint(result.processedLog(), tracepoint.getId(), snapshot.getID()); } - final Resource attributes = super.processAttributes(tracepoint); + final Collection metricDefinitions = tracepoint.getMetricDefinitions(); + if (!metricDefinitions.isEmpty()) { + for (MetricDefinition metricDefinition : metricDefinitions) { + processMetric(tracepoint, metricDefinition); + } + } + + final Resource attributes = processAttributes(tracepoint); snapshot.mergeAttributes(attributes); snapshots.add(snapshot); @@ -164,6 +175,49 @@ public Collection collect() { return snapshots; } + private void processMetric(final TracePointConfig tracepoint, final MetricDefinition metricDefinition) { + final IExpressionResult iExpressionResult = evaluateWatchExpression(metricDefinition.getExpression()); + final Number number = iExpressionResult.numberValue(); + if (iExpressionResult.isError() || Double.isNaN(number.doubleValue())) { + //todo how do we want to handle metrics that cannot be handled + return; + } + + final HashMap processedTags = new HashMap<>(); + final Map tags = metricDefinition.getTags(); + for (Entry entry : tags.entrySet()) { + final IExpressionResult tagResult = evaluateWatchExpression(entry.getValue()); + // todo check metric tag length + processedTags.put(entry.getKey(), Utils.truncate(tagResult.logString(), 200).value()); + } + + final IMetricProcessor metricProcessor = this.settings.getMetricProcessor(); + + try { + switch (metricDefinition.getType()) { + case "gauge": + metricProcessor.gauge(metricDefinition.getName(), processedTags, metricDefinition.getNamespace(), metricDefinition.getHelp(), + number.doubleValue()); + break; + case "counter": + metricProcessor.counter(metricDefinition.getName(), processedTags, metricDefinition.getNamespace(), metricDefinition.getHelp(), + number.doubleValue()); + break; + case "histogram": + metricProcessor.histogram(metricDefinition.getName(), processedTags, metricDefinition.getNamespace(), metricDefinition.getHelp(), + number.doubleValue()); + break; + case "summary": + metricProcessor.summary(metricDefinition.getName(), processedTags, metricDefinition.getNamespace(), metricDefinition.getHelp(), + number.doubleValue()); + break; + } + + } catch (Throwable t) { + t.printStackTrace(); + } + } + /** * {@inheritDoc} */ diff --git a/agent/src/main/java/com/intergral/deep/agent/tracepoint/inst/jsp/JSPMappedBreakpoint.java b/agent/src/main/java/com/intergral/deep/agent/tracepoint/inst/jsp/JSPMappedBreakpoint.java index 17708a0..c0d4357 100644 --- a/agent/src/main/java/com/intergral/deep/agent/tracepoint/inst/jsp/JSPMappedBreakpoint.java +++ b/agent/src/main/java/com/intergral/deep/agent/tracepoint/inst/jsp/JSPMappedBreakpoint.java @@ -28,7 +28,7 @@ public class JSPMappedBreakpoint extends TracePointConfig { public JSPMappedBreakpoint(final TracePointConfig tp, final int mappedLine) { - super(tp.getId(), tp.getPath(), tp.getLineNo(), tp.getArgs(), tp.getWatches()); + super(tp.getId(), tp.getPath(), tp.getLineNo(), tp.getArgs(), tp.getWatches(), tp.getMetricDefinitions()); this.mappedLine = mappedLine; } diff --git a/agent/src/main/java/com/intergral/deep/agent/types/TracePointConfig.java b/agent/src/main/java/com/intergral/deep/agent/types/TracePointConfig.java index 39903cd..f7a361b 100644 --- a/agent/src/main/java/com/intergral/deep/agent/types/TracePointConfig.java +++ b/agent/src/main/java/com/intergral/deep/agent/types/TracePointConfig.java @@ -17,6 +17,7 @@ package com.intergral.deep.agent.types; +import com.intergral.deep.agent.api.plugin.MetricDefinition; import com.intergral.deep.agent.settings.Settings; import java.util.Collection; import java.util.Map; @@ -84,25 +85,32 @@ public class TracePointConfig { private final TracepointWindow window; private final TracepointExecutionStats stats = new TracepointExecutionStats(); + private final Collection metricDefinitions; /** * Create a new tracepoint config. * - * @param id the id - * @param path the path - * @param lineNo the line number - * @param args the args - * @param watches the watches + * @param id the id + * @param path the path + * @param lineNo the line number + * @param args the args + * @param watches the watches + * @param metricDefinitions the metrics to evaluate */ - public TracePointConfig(final String id, final String path, final int lineNo, + public TracePointConfig( + final String id, + final String path, + final int lineNo, final Map args, - final Collection watches) { + final Collection watches, + final Collection metricDefinitions) { this.id = id; this.path = path; this.lineNo = lineNo; this.args = args; this.watches = watches; + this.metricDefinitions = metricDefinitions; this.window = new TracepointWindow(this.getArg(WINDOW_START, Integer.class, 0), this.getArg(WINDOW_END, Integer.class, 0)); } @@ -170,6 +178,14 @@ public Collection getWatches() { return watches; } + /** + * Get the tracepoint metric definitions + * @return the metric definitions + */ + public Collection getMetricDefinitions() { + return metricDefinitions; + } + /** * Get the tracepoint fire count. * diff --git a/agent/src/main/resources/META-INF/services/com.intergral.deep.agent.api.spi.MetricProvider b/agent/src/main/resources/META-INF/services/com.intergral.deep.agent.api.spi.MetricProvider new file mode 100644 index 0000000..7f05f9c --- /dev/null +++ b/agent/src/main/resources/META-INF/services/com.intergral.deep.agent.api.spi.MetricProvider @@ -0,0 +1,18 @@ +# +# Copyright (C) 2023 Intergral GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# + +com.intergral.deep.plugin.PrometheusMetricsPlugin diff --git a/agent/src/test/java/com/intergral/deep/agent/tracepoint/TracepointConfigServiceTest.java b/agent/src/test/java/com/intergral/deep/agent/tracepoint/TracepointConfigServiceTest.java index 16e0374..db96434 100644 --- a/agent/src/test/java/com/intergral/deep/agent/tracepoint/TracepointConfigServiceTest.java +++ b/agent/src/test/java/com/intergral/deep/agent/tracepoint/TracepointConfigServiceTest.java @@ -58,7 +58,7 @@ void noChange() { void customCallsUpdate() { // add a custom tracepoint will call instrumentation update final TracePointConfig tracePointConfig = tracepointConfigService.addCustom("/path", 123, Collections.emptyMap(), - Collections.emptyList()); + Collections.emptyList(), Collections.emptyList()); final ArgumentCaptor> captor = ArgumentCaptor.forClass(Collection.class); @@ -88,11 +88,11 @@ void customCallsUpdate() { void canLoadTracepointConfigs() { final TracePointConfig tracePointConfig1 = tracepointConfigService.addCustom("/path", 123, Collections.emptyMap(), - Collections.emptyList()); + Collections.emptyList(), Collections.emptyList()); final TracePointConfig tracePointConfig2 = tracepointConfigService.addCustom("/path", 123, Collections.emptyMap(), - Collections.emptyList()); + Collections.emptyList(), Collections.emptyList()); final TracePointConfig tracePointConfig3 = tracepointConfigService.addCustom("/path", 123, Collections.emptyMap(), - Collections.emptyList()); + Collections.emptyList(), Collections.emptyList()); { final Collection tracePointConfigs = tracepointConfigService.loadTracepointConfigs( @@ -121,7 +121,8 @@ void canLoadTracepointConfigs() { void configUpdate() { // TP config is passed to instrumentation tracepointConfigService.configUpdate(10101, "hash", - Collections.singletonList(new TracePointConfig("some-id", "path", 123, Collections.emptyMap(), Collections.emptyList()))); + Collections.singletonList( + new TracePointConfig("some-id", "path", 123, Collections.emptyMap(), Collections.emptyList(), Collections.emptyList()))); final ArgumentCaptor> captor = ArgumentCaptor.forClass(Collection.class); @@ -135,7 +136,7 @@ void configUpdate() { // custom and config are passed to instrumentation final TracePointConfig customTp = tracepointConfigService.addCustom("path", 123, Collections.emptyMap(), - Collections.emptyList()); + Collections.emptyList(), Collections.emptyList()); Mockito.verify(instrumentationService, Mockito.times(2)).processBreakpoints(captor.capture()); @@ -149,7 +150,8 @@ void configUpdate() { //custom remains after update tracepointConfigService.configUpdate(10101, "hash", - Collections.singletonList(new TracePointConfig("some-id", "path", 123, Collections.emptyMap(), Collections.emptyList()))); + Collections.singletonList( + new TracePointConfig("some-id", "path", 123, Collections.emptyMap(), Collections.emptyList(), Collections.emptyList()))); Mockito.verify(instrumentationService, Mockito.times(3)).processBreakpoints(captor.capture()); diff --git a/agent/src/test/java/com/intergral/deep/test/MockEventSnapshot.java b/agent/src/test/java/com/intergral/deep/test/MockEventSnapshot.java index 139a219..b50d77d 100644 --- a/agent/src/test/java/com/intergral/deep/test/MockEventSnapshot.java +++ b/agent/src/test/java/com/intergral/deep/test/MockEventSnapshot.java @@ -31,7 +31,7 @@ public class MockEventSnapshot extends EventSnapshot { public MockEventSnapshot() { - super(new TracePointConfig("tp-1", "/some/file/path.py", 123, new HashMap<>(), new ArrayList<>()), 1011L, Resource.DEFAULT, + super(new TracePointConfig("tp-1", "/some/file/path.py", 123, new HashMap<>(), new ArrayList<>(), new ArrayList<>()), 1011L, Resource.DEFAULT, new ArrayList<>(), new HashMap<>()); } diff --git a/agent/src/test/java/com/intergral/deep/test/MockTracepointConfig.java b/agent/src/test/java/com/intergral/deep/test/MockTracepointConfig.java index 5844623..56ce245 100644 --- a/agent/src/test/java/com/intergral/deep/test/MockTracepointConfig.java +++ b/agent/src/test/java/com/intergral/deep/test/MockTracepointConfig.java @@ -28,16 +28,16 @@ public class MockTracepointConfig extends TracePointConfig { public MockTracepointConfig() { - super("tp-id", "path", 123, new HashMap<>(), new ArrayList<>()); + super("tp-id", "path", 123, new HashMap<>(), new ArrayList<>(), new ArrayList<>()); } public MockTracepointConfig(final String path) { - super("tp-id", path, 123, new HashMap<>(), new ArrayList<>()); + super("tp-id", path, 123, new HashMap<>(), new ArrayList<>(), new ArrayList<>()); } public MockTracepointConfig(final String path, final int line) { - super("tp-id", path, line, new HashMap<>(), new ArrayList<>()); + super("tp-id", path, line, new HashMap<>(), new ArrayList<>(), new ArrayList<>()); } public MockTracepointConfig withArg(final String key, final String value) { diff --git a/deep/pom.xml b/deep/pom.xml index 652e284..3ddee37 100644 --- a/deep/pom.xml +++ b/deep/pom.xml @@ -69,6 +69,12 @@ 1.0-SNAPSHOT compile + + com.intergral.deep.plugins + prometheus-metrics + 1.0-SNAPSHOT + compile + @@ -127,27 +133,43 @@ + ${git.branch} + ${git.commit.id} + ${git.commit.time} + ${git.dirty} + ${git.tags} + ${git.remote.origin.url} ${maven.build.timestamp} + ${parsedVersion.majorVersion} + ${parsedVersion.minorVersion} + ${parsedVersion.incrementalVersion} ${project.version} + ${git.commit.id} + ${git.branch} + ${env.CI_PROJECT_NAME} + ${env.CI_PIPELINE_ID} + ${env.CI_PIPELINE_IID} + ${env.CI_PIPELINE_SOURCE} + ${env.CI_PIPELINE_URL} ${project.artifactId} diff --git a/examples/agent-load/src/main/java/com/intergral/deep/examples/Main.java b/examples/agent-load/src/main/java/com/intergral/deep/examples/Main.java index a64caf1..6472192 100644 --- a/examples/agent-load/src/main/java/com/intergral/deep/examples/Main.java +++ b/examples/agent-load/src/main/java/com/intergral/deep/examples/Main.java @@ -57,7 +57,7 @@ public static void main(String[] args) throws Throwable { // USe the API to create a tracepoint that will fire forever DeepAPI.api() .registerTracepoint("com/intergral/deep/examples/SimpleTest", 46, Collections.singletonMap("fire_count", "-1"), - Collections.emptyList()); + Collections.emptyList(), Collections.emptyList()); final SimpleTest ts = new SimpleTest("This is a test", 2); for (; ; ) { diff --git a/examples/dynamic-load/src/main/java/com/intergral/deep/examples/Main.java b/examples/dynamic-load/src/main/java/com/intergral/deep/examples/Main.java index 241a101..da7ae2c 100644 --- a/examples/dynamic-load/src/main/java/com/intergral/deep/examples/Main.java +++ b/examples/dynamic-load/src/main/java/com/intergral/deep/examples/Main.java @@ -53,7 +53,7 @@ public static void main(String[] args) throws Throwable { .getParent() .getParent() .getParent() - .resolve("agent/target/deep-1.0-SNAPSHOT.jar"); + .resolve("agent/target/agent-1.0-SNAPSHOT.jar"); // Dynamically configure and attach the deep agent Deep.config() @@ -81,7 +81,7 @@ public static void main(String[] args) throws Throwable { // USe the API to create a tracepoint that will fire forever DeepAPI.api() .registerTracepoint("com/intergral/deep/examples/SimpleTest", 46, - Collections.singletonMap("fire_count", "-1"), Collections.emptyList()); + Collections.singletonMap("fire_count", "-1"), Collections.emptyList(), Collections.emptyList()); final SimpleTest ts = new SimpleTest("This is a test", 2); for (; ; ) { diff --git a/examples/pom.xml b/examples/pom.xml index f309ab1..d483fcd 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -37,6 +37,7 @@ agent-load dynamic-load + prometheus-metrics-example diff --git a/examples/prometheus-metrics-example/pom.xml b/examples/prometheus-metrics-example/pom.xml new file mode 100644 index 0000000..bbb081e --- /dev/null +++ b/examples/prometheus-metrics-example/pom.xml @@ -0,0 +1,70 @@ + + + + + 4.0.0 + + com.intergral.deep.examples + examples + 1.0-SNAPSHOT + + + prometheus-metrics-example + + + + + + + + com.intergral.deep + deep + 1.0-SNAPSHOT + compile + + + + com.intergral.deep + agent + 1.0-SNAPSHOT + provided + + + com.intergral.deep.plugins + prometheus-metrics + 1.0-SNAPSHOT + + + + + io.prometheus + prometheus-metrics-core + 1.1.0 + + + io.prometheus + prometheus-metrics-exporter-httpserver + 1.1.0 + + + + \ No newline at end of file diff --git a/examples/prometheus-metrics-example/src/main/java/com/intergral/deep/examples/BaseTest.java b/examples/prometheus-metrics-example/src/main/java/com/intergral/deep/examples/BaseTest.java new file mode 100644 index 0000000..1a3ba44 --- /dev/null +++ b/examples/prometheus-metrics-example/src/main/java/com/intergral/deep/examples/BaseTest.java @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2023 Intergral GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.intergral.deep.examples; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.UUID; + +public class BaseTest { + + protected final Properties systemProps = System.getProperties(); + + + public String newId() { + return UUID.randomUUID().toString(); + } + + + public Map makeCharCountMap(final String str) { + final HashMap res = new HashMap(); + + for (int i = 0; i < str.length(); i++) { + final char c = str.charAt(i); + final Integer cnt = res.get(c); + if (cnt == null) { + res.put(c, 0); + } else { + res.put(c, cnt + 1); + } + } + + return res; + } +} diff --git a/examples/prometheus-metrics-example/src/main/java/com/intergral/deep/examples/Main.java b/examples/prometheus-metrics-example/src/main/java/com/intergral/deep/examples/Main.java new file mode 100644 index 0000000..8a093d4 --- /dev/null +++ b/examples/prometheus-metrics-example/src/main/java/com/intergral/deep/examples/Main.java @@ -0,0 +1,135 @@ +/* + * Copyright (C) 2023 Intergral GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.intergral.deep.examples; + + +import com.intergral.deep.Deep; +import com.intergral.deep.agent.api.IDeep; +import com.intergral.deep.agent.api.plugin.MetricDefinition; +import com.intergral.deep.agent.api.reflection.IReflection; +import com.intergral.deep.agent.api.resource.Resource; +import com.intergral.deep.agent.api.settings.ISettings; +import com.intergral.deep.api.DeepAPI; +import io.prometheus.metrics.core.metrics.Histogram; +import io.prometheus.metrics.exporter.httpserver.HTTPServer; +import io.prometheus.metrics.model.snapshots.Unit; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; + +/** + * This example expects the deep agent to be loaded via the javaagent vm option. + *

+ * See RunConfigurations for IDEA: + *

    + *
  • Dynamic Load without JavaAgent
    • + *
  • Dynamic Load with JavaAgent
    • + *
+ */ +public class Main { + + /** + * Main entry for example. + * + * @param args the startup arguments + * @throws Throwable if we error + */ + public static void main(String[] args) throws Throwable { + // this is only needed in this example as we are using a local built module + // if using the dependency from maven you do not need to set the path + Path jarPath = Paths.get(Main.class.getResource("/").toURI()) + .getParent() + .getParent() + .getParent() + .getParent() + .resolve("agent/target/agent-1.0-SNAPSHOT.jar"); + + // Dynamically configure and attach the deep agent + Deep.config() + .setJarPath(jarPath.toAbsolutePath().toString()) + .setValue(ISettings.KEY_SERVICE_URL, "localhost:43315") + .setValue(ISettings.KEY_SERVICE_SECURE, false) + .start(); + + // different ways to get the API instance + final Deep instance = Deep.getInstance(); + System.out.println(instance.api().getVersion()); + System.out.println(instance.reflection()); + + System.out.println(DeepAPI.api().getVersion()); + System.out.println(DeepAPI.reflection()); + + // Use the API to register a plugin + // This plugin will attach the attribute 'example' to the created snapshot + // you should also see the log line 'custom plugin' when you run this example + DeepAPI.api().registerPlugin((settings, snapshot) -> { + System.out.println("custom plugin"); + return Resource.create(Collections.singletonMap("example", "dynamic_load")); + }); + +// final Class aClass = Class.forName("com.intergral.deep.plugin.PrometheusMetricsPlugin"); +// final Constructor constructor = aClass.getConstructor(); +// final Object o = constructor.newInstance(); +// +// DeepAPI.api().registerPlugin((IPlugin) o); + + Histogram histogram = Histogram.builder() + .name("request_latency_seconds") + .help("request latency in seconds") + .unit(Unit.SECONDS) + .labelNames("path", "status") + .register(); + + HTTPServer server = HTTPServer.builder() + .port(9400) + .buildAndStart(); + + System.out.println("HTTPServer listening on port http://localhost:" + server.getPort() + "/metrics"); + + final HashMap tags = new HashMap<>(); + // USe the API to create a tracepoint that will fire forever + final Map fireCount = new HashMap<>(); + fireCount.put("fire_count", "-1"); + fireCount.put("log_msg", "This is a log message {this}"); + + DeepAPI.api() + .registerTracepoint("com/intergral/deep/examples/SimpleTest", 46, + fireCount, Collections.emptyList(), + Collections.singletonList(new MetricDefinition("custom_metric", tags, "histogram", "this.cnt", "deep", "help message"))); + + Random random = new Random(0); + final SimpleTest ts = new SimpleTest("This is a test", 2); + for (; ; ) { + try { + ts.message(ts.newId()); + } catch (Exception e) { + e.printStackTrace(); + } + + double duration = Math.abs(random.nextGaussian() / 10.0 + 0.2); + String status = random.nextInt(100) < 20 ? "500" : "200"; + histogram.labelValues("/", status).observe(duration); + + Thread.sleep(1000); + } + + } +} diff --git a/examples/prometheus-metrics-example/src/main/java/com/intergral/deep/examples/SimpleTest.java b/examples/prometheus-metrics-example/src/main/java/com/intergral/deep/examples/SimpleTest.java new file mode 100644 index 0000000..760235b --- /dev/null +++ b/examples/prometheus-metrics-example/src/main/java/com/intergral/deep/examples/SimpleTest.java @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2023 Intergral GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.intergral.deep.examples; + +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; + +public class SimpleTest extends BaseTest { + + public static Date NICE_START_DATE = new Date(); + + private final long startedAt = System.currentTimeMillis(); + private final String testName; + private final int maxExecutions; + public int cnt = 0; + private Map charCounter = new TreeMap(); + + + public SimpleTest(final String testName, final int maxExecutions) { + this.testName = testName; + this.maxExecutions = maxExecutions; + } + + + void message(final String uuid) throws Exception { + System.out.println(cnt + ":" + uuid); + cnt += 1; + + checkEnd(cnt, maxExecutions); + + final Map info = makeCharCountMap(uuid); + merge(charCounter, info); + if ((cnt % 30) == 0) { + dump(); + } + } + + + void merge(final Map charCounter, final Map newInfo) { + for (final Character c : newInfo.keySet()) { + final Integer i = newInfo.get(c); + + Integer curr = charCounter.get(c); + if (curr == null) { + charCounter.put(c, i); + } else { + charCounter.put(c, curr + i); + } + } + } + + + void dump() { + System.out.println(charCounter); + charCounter = new HashMap(); + } + + + static void checkEnd(final int val, final int max) throws Exception { + if (val > max) { + throw new Exception("Hit max executions " + val + " " + max); + } + } + + + @Override + public String toString() { + return getClass().getName() + ":" + testName + ":" + startedAt + "#" + System.identityHashCode( + this); + } +} diff --git a/plugins/pom.xml b/plugins/pom.xml index 1607ccc..47f43ab 100644 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -30,6 +30,7 @@ java-plugin cf-plugin + prometheus-metrics com.intergral.deep.plugins diff --git a/plugins/prometheus-metrics/pom.xml b/plugins/prometheus-metrics/pom.xml new file mode 100644 index 0000000..78b37ee --- /dev/null +++ b/plugins/prometheus-metrics/pom.xml @@ -0,0 +1,50 @@ + + + + + 4.0.0 + + com.intergral.deep.plugins + plugins + 1.0-SNAPSHOT + + + prometheus-metrics + + + 1.1.0 + + + + + com.intergral.deep + agent-api + 1.0-SNAPSHOT + compile + + + io.prometheus + prometheus-metrics-core + ${prometheus.version} + provided + + + + \ No newline at end of file diff --git a/plugins/prometheus-metrics/src/main/java/com/intergral/deep/plugin/PrometheusMetricsPlugin.java b/plugins/prometheus-metrics/src/main/java/com/intergral/deep/plugin/PrometheusMetricsPlugin.java new file mode 100644 index 0000000..00f1021 --- /dev/null +++ b/plugins/prometheus-metrics/src/main/java/com/intergral/deep/plugin/PrometheusMetricsPlugin.java @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2023 Intergral GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.intergral.deep.plugin; + +import com.intergral.deep.agent.api.spi.ConditionalProvider; +import com.intergral.deep.agent.api.spi.MetricProvider; +import io.prometheus.metrics.core.metrics.Counter; +import io.prometheus.metrics.core.metrics.Counter.Builder; +import io.prometheus.metrics.core.metrics.Gauge; +import io.prometheus.metrics.core.metrics.Histogram; +import io.prometheus.metrics.core.metrics.Summary; +import io.prometheus.metrics.model.registry.PrometheusRegistry; +import java.util.Arrays; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +public class PrometheusMetricsPlugin implements MetricProvider, ConditionalProvider { + + private static final Map REGISTRY_CACHE = new ConcurrentHashMap<>(); + + @Override + public boolean isActive() { + try { + // just trying to see if the prometheus classes are loaded + final PrometheusRegistry ignored = PrometheusRegistry.defaultRegistry; + return true; + } catch (Throwable t) { + return false; + } + } + + @Override + public void counter(final String name, final Map tags, final String namespace, final String help, final Double value) { + final String ident = buildIdent("counter", namespace, name, tags.keySet()); + Object o = REGISTRY_CACHE.get(ident); + if (o == null) { + + final Builder builder = Counter.builder().name(String.format("%s_%s", namespace, name)).help(help); + if (!tags.isEmpty()) { + builder.labelNames(Arrays.toString(tags.keySet().toArray())); + } + final Counter register = builder.register(); + REGISTRY_CACHE.put(ident, register); + + o = register; + } + + if (!tags.isEmpty()) { + ((Counter) o).labelValues(Arrays.toString(tags.values().toArray())).inc(value); + } else { + ((Counter) o).inc(value); + } + } + + @Override + public void gauge(final String name, final Map tags, final String namespace, final String help, final Double value) { + final String ident = buildIdent("gauge", namespace, name, tags.keySet()); + Object o = REGISTRY_CACHE.get(ident); + if (o == null) { + final Gauge.Builder builder = Gauge.builder().name(String.format("%s_%s", namespace, name)).help(help); + if (!tags.isEmpty()) { + builder.labelNames(Arrays.toString(tags.keySet().toArray())); + } + final Gauge register = builder.register(); + REGISTRY_CACHE.put(ident, register); + + o = register; + } + + if (!tags.isEmpty()) { + ((Gauge) o).labelValues(Arrays.toString(tags.values().toArray())).set(value); + } else { + ((Gauge) o).set(value); + } + } + + @Override + public void histogram(final String name, final Map tags, final String namespace, final String help, final Double value) { + final String ident = buildIdent("histogram", namespace, name, tags.keySet()); + Object o = REGISTRY_CACHE.get(ident); + if (o == null) { + final Histogram.Builder builder = Histogram.builder().name(String.format("%s_%s", namespace, name)).help(help); + if (!tags.isEmpty()) { + builder.labelNames(Arrays.toString(tags.keySet().toArray())); + } + final Histogram register = builder.register(); + REGISTRY_CACHE.put(ident, register); + o = register; + } + + if (!tags.isEmpty()) { + ((Histogram) o).labelValues(Arrays.toString(tags.values().toArray())).observe(value); + } else { + ((Histogram) o).observe(value); + } + } + + @Override + public void summary(final String name, final Map tags, final String namespace, final String help, final Double value) { + final String ident = buildIdent("summary", namespace, name, tags.keySet()); + Object o = REGISTRY_CACHE.get(ident); + if (o == null) { + final Summary.Builder builder = Summary.builder().name(String.format("%s_%s", namespace, name)).help(help); + if (!tags.isEmpty()) { + builder.labelNames(Arrays.toString(tags.keySet().toArray())); + } + final Summary register = builder.register(); + REGISTRY_CACHE.put(ident, register); + o = register; + } + + if (!tags.isEmpty()) { + ((Summary) o).labelValues(Arrays.toString(tags.values().toArray())).observe(value); + } else { + ((Summary) o).observe(value); + } + } + + private String buildIdent(final String type, final String namespace, final String name, final Set labelNames) { + return String.format("%s_%s_%s_%s", type, namespace, name, String.join("-", labelNames)); + } +} diff --git a/pom.xml b/pom.xml index f4185f0..35675a6 100644 --- a/pom.xml +++ b/pom.xml @@ -85,6 +85,7 @@ + 0.0.1-SNAPSHOT 1.59.0 4.1.98.Final 2.0.62.Final @@ -452,7 +453,7 @@ com.intergral.deep deep-proto - 1.0.2 + ${deep-proto.version} From 0ad6ac02cbb46537deea954ab33ac24f2aa72e20 Mon Sep 17 00:00:00 2001 From: Ben Donnelly Date: Thu, 14 Dec 2023 10:38:19 +0000 Subject: [PATCH 2/4] feat(metrics): update metrics to use new plugin system --- .idea/inspectionProfiles/Project_Default.xml | 3 + CHANGELOG.md | 1 + .../com/intergral/deep/agent/api/IDeep.java | 2 - .../agent/api/plugin/IMetricProcessor.java | 56 ++++++++++- .../agent/api/plugin/MetricDefinition.java | 56 ++++++++++- .../agent/api/spi/ConditionalProvider.java | 22 ----- .../deep/agent/api/spi/MetricProvider.java | 24 ----- .../api/plugin/MetricDefinitionTest.java | 93 +++++++++++++++++++ .../com/intergral/deep/agent/DeepAgent.java | 4 - .../deep/agent/grpc/GrpcService.java | 17 +--- .../deep/agent/poll/LongPollService.java | 6 +- .../deep/agent/push/PushService.java | 19 ++-- .../deep/agent/resource/SpiUtil.java | 12 --- .../deep/agent/settings/Settings.java | 52 ++++++++--- .../tracepoint/handler/FrameProcessor.java | 81 ++++++++++------ .../deep/agent/types/TracePointConfig.java | 3 +- .../deep/agent/poll/LongPollServiceTest.java | 35 ++++++- .../deep/agent/settings/SettingsTest.java | 62 +++++++++++++ .../handler/FrameCollectorTest.java | 6 ++ .../handler/FrameProcessorTest.java | 21 +++++ .../deep/test/MockEventSnapshot.java | 4 +- .../deep/test/MockTracepointConfig.java | 7 ++ ...m.intergral.deep.agent.api.spi.IDeepPlugin | 0 .../com/intergral/deep/examples/Main.java | 80 +++++++--------- .../deep/plugin/PrometheusMetricsPlugin.java | 25 +++-- 25 files changed, 499 insertions(+), 192 deletions(-) delete mode 100644 agent-api/src/main/java/com/intergral/deep/agent/api/spi/ConditionalProvider.java delete mode 100644 agent-api/src/main/java/com/intergral/deep/agent/api/spi/MetricProvider.java create mode 100644 agent-api/src/test/java/com/intergral/deep/agent/api/plugin/MetricDefinitionTest.java rename agent/src/main/resources/META-INF/services/com.intergral.deep.agent.api.spi.MetricProvider => deep/src/main/java/resources/META-INF/services/com.intergral.deep.agent.api.spi.IDeepPlugin (100%) diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml index c0ee0d5..d1d8003 100644 --- a/.idea/inspectionProfiles/Project_Default.xml +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -1,6 +1,9 @@