[ML] Add created_by info to usage stats (#40518)

This change adds information about which UI path
(if any) created ML anomaly detector jobs to the
stats returned by the _xpack/usage endpoint.

Counts for the following possibilities are expected:

* ml_module_apache_access
* ml_module_apm_transaction
* ml_module_auditbeat_process_docker
* ml_module_auditbeat_process_hosts
* ml_module_nginx_access
* ml_module_sample
* multi_metric_wizard
* population_wizard
* single_metric_wizard
* unknown

The "unknown" count is for jobs that do not have a
created_by setting in their custom_settings.

Closes #38403

Author: David Roberts

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MachineLearningFeatureSetUsage.java

@@ -25,6 +25,7 @@ public class MachineLearningFeatureSetUsage extends XPackFeatureSet.Usage {
     public static final String DETECTORS = "detectors";
     public static final String FORECASTS = "forecasts";
     public static final String MODEL_SIZE = "model_size";
+    public static final String CREATED_BY = "created_by";
     public static final String NODE_COUNT = "node_count";
 
     private final Map<String, Object> jobsUsage;

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSet.java

@@ -216,13 +216,16 @@ private void addJobsUsage(GetJobsStatsAction.Response response, List<Job> jobs)
             Map<JobState, StatsAccumulator> detectorStatsByState = new HashMap<>();
             Map<JobState, StatsAccumulator> modelSizeStatsByState = new HashMap<>();
             Map<JobState, ForecastStats> forecastStatsByState = new HashMap<>();
+            Map<JobState, Map<String, Long>> createdByByState = new HashMap<>();
 
             List<GetJobsStatsAction.Response.JobStats> jobsStats = response.getResponse().results();
             Map<String, Job> jobMap = jobs.stream().collect(Collectors.toMap(Job::getId, item -> item));
+            Map<String, Long> allJobsCreatedBy = jobs.stream().map(this::jobCreatedBy)
+                    .collect(Collectors.groupingBy(item -> item, Collectors.counting()));;
             for (GetJobsStatsAction.Response.JobStats jobStats : jobsStats) {
                 ModelSizeStats modelSizeStats = jobStats.getModelSizeStats();
-                int detectorsCount = jobMap.get(jobStats.getJobId()).getAnalysisConfig()
-                        .getDetectors().size();
+                Job job = jobMap.get(jobStats.getJobId());
+                int detectorsCount = job.getAnalysisConfig().getDetectors().size();
                 double modelSize = modelSizeStats == null ? 0.0
                         : jobStats.getModelSizeStats().getModelBytes();
 
@@ -237,27 +240,41 @@ private void addJobsUsage(GetJobsStatsAction.Response response, List<Job> jobs)
                 modelSizeStatsByState.computeIfAbsent(jobState,
                         js -> new StatsAccumulator()).add(modelSize);
                 forecastStatsByState.merge(jobState, jobStats.getForecastStats(), (f1, f2) -> f1.merge(f2));
+                createdByByState.computeIfAbsent(jobState, js -> new HashMap<>())
+                        .compute(jobCreatedBy(job), (k, v) -> (v == null) ? 1L : (v + 1));
             }
 
             jobsUsage.put(MachineLearningFeatureSetUsage.ALL, createJobUsageEntry(jobs.size(), allJobsDetectorsStats,
-                    allJobsModelSizeStats, allJobsForecastStats));
+                    allJobsModelSizeStats, allJobsForecastStats, allJobsCreatedBy));
             for (JobState jobState : jobCountByState.keySet()) {
                 jobsUsage.put(jobState.name().toLowerCase(Locale.ROOT), createJobUsageEntry(
                         jobCountByState.get(jobState).get(),
                         detectorStatsByState.get(jobState),
                         modelSizeStatsByState.get(jobState),
-                        forecastStatsByState.get(jobState)));
+                        forecastStatsByState.get(jobState),
+                        createdByByState.get(jobState)));
             }
         }
 
+        private String jobCreatedBy(Job job) {
+            Map<String, Object> customSettings = job.getCustomSettings();
+            if (customSettings == null || customSettings.containsKey(MachineLearningFeatureSetUsage.CREATED_BY) == false) {
+                return "unknown";
+            }
+            // Replace non-alpha-numeric characters with underscores because
+            // the values from custom settings become keys in the usage data
+            return customSettings.get(MachineLearningFeatureSetUsage.CREATED_BY).toString().replaceAll("\\W", "_");
+        }
+
         private Map<String, Object> createJobUsageEntry(long count, StatsAccumulator detectorStats,
                                                         StatsAccumulator modelSizeStats,
-                                                        ForecastStats forecastStats) {
+                                                        ForecastStats forecastStats, Map<String, Long> createdBy) {
             Map<String, Object> usage = new HashMap<>();
             usage.put(MachineLearningFeatureSetUsage.COUNT, count);
             usage.put(MachineLearningFeatureSetUsage.DETECTORS, detectorStats.asMap());
             usage.put(MachineLearningFeatureSetUsage.MODEL_SIZE, modelSizeStats.asMap());
             usage.put(MachineLearningFeatureSetUsage.FORECASTS, forecastStats.asMap());
+            usage.put(MachineLearningFeatureSetUsage.CREATED_BY, createdBy);
             return usage;
         }
 

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSetTests.java

@@ -155,7 +155,8 @@ public void testUsage() throws Exception {
         Settings.Builder settings = Settings.builder().put(commonSettings);
         settings.put("xpack.ml.enabled", true);
 
-        Job opened1 = buildJob("opened1", Arrays.asList(buildMinDetector("foo")));
+        Job opened1 = buildJob("opened1", Collections.singletonList(buildMinDetector("foo")),
+                Collections.singletonMap("created_by", randomFrom("a-cool-module", "a_cool_module", "a cool module")));
         GetJobsStatsAction.Response.JobStats opened1JobStats = buildJobStats("opened1", JobState.OPENED, 100L, 3L);
         Job opened2 = buildJob("opened2", Arrays.asList(buildMinDetector("foo"), buildMinDetector("bar")));
         GetJobsStatsAction.Response.JobStats opened2JobStats = buildJobStats("opened2", JobState.OPENED, 200L, 8L);
@@ -200,6 +201,8 @@ public void testUsage() throws Exception {
             assertThat(source.getValue("jobs._all.model_size.max"), equalTo(300.0));
             assertThat(source.getValue("jobs._all.model_size.total"), equalTo(600.0));
             assertThat(source.getValue("jobs._all.model_size.avg"), equalTo(200.0));
+            assertThat(source.getValue("jobs._all.created_by.a_cool_module"), equalTo(1));
+            assertThat(source.getValue("jobs._all.created_by.unknown"), equalTo(2));
 
             assertThat(source.getValue("jobs.opened.count"), equalTo(2));
             assertThat(source.getValue("jobs.opened.detectors.min"), equalTo(1.0));
@@ -210,6 +213,8 @@ public void testUsage() throws Exception {
             assertThat(source.getValue("jobs.opened.model_size.max"), equalTo(200.0));
             assertThat(source.getValue("jobs.opened.model_size.total"), equalTo(300.0));
             assertThat(source.getValue("jobs.opened.model_size.avg"), equalTo(150.0));
+            assertThat(source.getValue("jobs.opened.created_by.a_cool_module"), equalTo(1));
+            assertThat(source.getValue("jobs.opened.created_by.unknown"), equalTo(1));
 
             assertThat(source.getValue("jobs.closed.count"), equalTo(1));
             assertThat(source.getValue("jobs.closed.detectors.min"), equalTo(3.0));
@@ -220,6 +225,8 @@ public void testUsage() throws Exception {
             assertThat(source.getValue("jobs.closed.model_size.max"), equalTo(300.0));
             assertThat(source.getValue("jobs.closed.model_size.total"), equalTo(300.0));
             assertThat(source.getValue("jobs.closed.model_size.avg"), equalTo(300.0));
+            assertThat(source.getValue("jobs.closed.created_by.a_cool_module"), is(nullValue()));
+            assertThat(source.getValue("jobs.closed.created_by.unknown"), equalTo(1));
 
             assertThat(source.getValue("jobs.opening"), is(nullValue()));
             assertThat(source.getValue("jobs.closing"), is(nullValue()));
@@ -359,6 +366,7 @@ private void givenJobs(List<Job> jobs, List<GetJobsStatsAction.Response.JobStats
         }).when(jobManager).expandJobs(eq(MetaData.ALL), eq(true), any(ActionListener.class));
 
         doAnswer(invocationOnMock -> {
+            @SuppressWarnings("unchecked")
             ActionListener<GetJobsStatsAction.Response> listener =
                     (ActionListener<GetJobsStatsAction.Response>) invocationOnMock.getArguments()[2];
             listener.onResponse(new GetJobsStatsAction.Response(
@@ -400,6 +408,7 @@ private void givenNodeCount(int nodeCount) {
 
     private void givenDatafeeds(List<GetDatafeedsStatsAction.Response.DatafeedStats> datafeedStats) {
         doAnswer(invocationOnMock -> {
+            @SuppressWarnings("unchecked")
             ActionListener<GetDatafeedsStatsAction.Response> listener =
                     (ActionListener<GetDatafeedsStatsAction.Response>) invocationOnMock.getArguments()[2];
             listener.onResponse(new GetDatafeedsStatsAction.Response(
@@ -416,10 +425,15 @@ private static Detector buildMinDetector(String fieldName) {
     }
 
     private static Job buildJob(String jobId, List<Detector> detectors) {
+        return buildJob(jobId, detectors, null);
+    }
+
+    private static Job buildJob(String jobId, List<Detector> detectors, Map<String, Object> customSettings) {
         AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(detectors);
         return new Job.Builder(jobId)
                 .setAnalysisConfig(analysisConfig)
                 .setDataDescription(new DataDescription.Builder())
+                .setCustomSettings(customSettings)
                 .build(new Date(randomNonNegativeLong()));
     }