From cb2eece008ea300a38a3050f98a747575053d051 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 11 Feb 2019 16:24:11 +0100 Subject: [PATCH 1/4] set meta data on data frame index --- .../xpack/core/dataframe/DataFrameField.java | 7 +++++++ .../dataframe/persistence/DataframeIndex.java | 21 +++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java index 41852aa972900..7ee7a0c155cde 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java @@ -29,6 +29,13 @@ public final class DataFrameField { // note: this is used to match tasks public static final String PERSISTENT_TASK_DESCRIPTION_PREFIX = "data_frame_"; + // strings for meta information + public static final String META_FIELDNAME = "_data_frame"; + public static final String CREATION_DATE_MILLIS = "creation_date_in_millis"; + public static final String VERSION = "version"; + public static final String CREATED = "created"; + public static final String TRANSFORM = "transform"; + private DataFrameField() { } } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndex.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndex.java index 758027694f458..305698e43cf44 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndex.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndex.java @@ -8,12 +8,14 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.client.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.dataframe.DataFrameField; import org.elasticsearch.xpack.core.dataframe.DataFrameMessages; import org.elasticsearch.xpack.dataframe.transforms.DataFrameTransformConfig; @@ -29,6 +31,7 @@ public final class DataframeIndex { public static final String DOC_TYPE = "_doc"; private static final String PROPERTIES = "properties"; private static final String TYPE = "type"; + private static final String META = "_meta"; private DataframeIndex() { } @@ -41,7 +44,7 @@ public static void createDestinationIndex(Client client, DataFrameTransformConfi request.settings(Settings.builder() // <1> .put("index.number_of_shards", 1).put("index.number_of_replicas", 0)); - request.mapping(DOC_TYPE, createMappingXContent(mappings)); + request.mapping(DOC_TYPE, createMappingXContent(mappings, transformConfig.getId())); client.execute(CreateIndexAction.INSTANCE, request, ActionListener.wrap(createIndexResponse -> { listener.onResponse(true); @@ -53,10 +56,11 @@ public static void createDestinationIndex(Client client, DataFrameTransformConfi })); } - private static XContentBuilder createMappingXContent(Map mappings) { + private static XContentBuilder createMappingXContent(Map mappings, String id) { try { XContentBuilder builder = jsonBuilder().startObject(); builder.startObject(DOC_TYPE); + addMetaData(builder, id); builder.startObject(PROPERTIES); for (Entry field : mappings.entrySet()) { builder.startObject(field.getKey()).field(TYPE, field.getValue()).endObject(); @@ -68,4 +72,17 @@ private static XContentBuilder createMappingXContent(Map mapping throw new RuntimeException(e); } } + + private static XContentBuilder addMetaData(XContentBuilder builder, String id) throws IOException { + builder.startObject(META); + builder.startObject(DataFrameField.META_FIELDNAME); + builder.field(DataFrameField.CREATION_DATE_MILLIS, System.currentTimeMillis()); + builder.startObject(DataFrameField.VERSION); + builder.field(DataFrameField.CREATED, Version.CURRENT); + builder.endObject(); + builder.field(DataFrameField.TRANSFORM, id); + builder.endObject(); // META_FIELDNAME + builder.endObject(); // META + return builder; + } } From 36fc25d38afdd69a5d9ab26ef62eaecce2920129 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Tue, 12 Feb 2019 11:37:45 +0100 Subject: [PATCH 2/4] add a created_by field --- .../org/elasticsearch/xpack/core/dataframe/DataFrameField.java | 2 ++ .../xpack/dataframe/persistence/DataframeIndex.java | 1 + 2 files changed, 3 insertions(+) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java index 7ee7a0c155cde..d2956e6559b0e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/DataFrameField.java @@ -34,7 +34,9 @@ public final class DataFrameField { public static final String CREATION_DATE_MILLIS = "creation_date_in_millis"; public static final String VERSION = "version"; public static final String CREATED = "created"; + public static final String CREATED_BY = "created_by"; public static final String TRANSFORM = "transform"; + public static final String DATA_FRAME_SIGNATURE = "data-frame-transform"; private DataFrameField() { } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndex.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndex.java index 305698e43cf44..cbca203436aeb 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndex.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndex.java @@ -77,6 +77,7 @@ private static XContentBuilder addMetaData(XContentBuilder builder, String id) t builder.startObject(META); builder.startObject(DataFrameField.META_FIELDNAME); builder.field(DataFrameField.CREATION_DATE_MILLIS, System.currentTimeMillis()); + builder.field(DataFrameField.CREATED_BY, DataFrameField.DATA_FRAME_SIGNATURE); builder.startObject(DataFrameField.VERSION); builder.field(DataFrameField.CREATED, Version.CURRENT); builder.endObject(); From b99e565ec4ba7dd2c780c54a7af003021febb265 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Tue, 12 Feb 2019 11:38:05 +0100 Subject: [PATCH 3/4] add a IT for meta data --- .../integration/DataFrameMetaDataIT.java | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameMetaDataIT.java diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameMetaDataIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameMetaDataIT.java new file mode 100644 index 0000000000000..ddaeb2d33744a --- /dev/null +++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameMetaDataIT.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.dataframe.integration; + +import org.elasticsearch.Version; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.junit.Before; + +import java.io.IOException; +import java.util.Map; + +public class DataFrameMetaDataIT extends DataFrameRestTestCase { + + private boolean indicesCreated = false; + + // preserve indices in order to reuse source indices in several test cases + @Override + protected boolean preserveIndicesUponCompletion() { + return true; + } + + @Before + public void createIndexes() throws IOException { + + // it's not possible to run it as @BeforeClass as clients aren't initialized then, so we need this little hack + if (indicesCreated) { + return; + } + + createReviewsIndex(); + indicesCreated = true; + } + + public void testMetaData() throws IOException { + long testStarted = System.currentTimeMillis(); + createPivotReviewsTransform("test_meta", "pivot_reviews", null); + + Response mappingResponse = client().performRequest(new Request("GET", "pivot_reviews/_mapping")); + + Map mappingAsMap = entityAsMap(mappingResponse); + assertEquals(Version.CURRENT.toString(), + XContentMapValues.extractValue("pivot_reviews.mappings._meta._data_frame.version.created", mappingAsMap)); + assertTrue((Long) XContentMapValues.extractValue("pivot_reviews.mappings._meta._data_frame.creation_date_in_millis", + mappingAsMap) < System.currentTimeMillis()); + assertTrue((Long) XContentMapValues.extractValue("pivot_reviews.mappings._meta._data_frame.creation_date_in_millis", + mappingAsMap) > testStarted); + assertEquals("test_meta", + XContentMapValues.extractValue("pivot_reviews.mappings._meta._data_frame.transform", mappingAsMap)); + assertEquals("data-frame-transform", + XContentMapValues.extractValue("pivot_reviews.mappings._meta._data_frame.created_by", mappingAsMap)); + } + +} From 83f029bd58a09f4140ccbd995db5283b33e662ed Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Wed, 13 Feb 2019 21:17:40 +0100 Subject: [PATCH 4/4] move created_by one level up --- .../xpack/dataframe/integration/DataFrameMetaDataIT.java | 2 +- .../xpack/dataframe/persistence/DataframeIndex.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameMetaDataIT.java b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameMetaDataIT.java index ddaeb2d33744a..d278c78842c39 100644 --- a/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameMetaDataIT.java +++ b/x-pack/plugin/data-frame/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/dataframe/integration/DataFrameMetaDataIT.java @@ -53,7 +53,7 @@ public void testMetaData() throws IOException { assertEquals("test_meta", XContentMapValues.extractValue("pivot_reviews.mappings._meta._data_frame.transform", mappingAsMap)); assertEquals("data-frame-transform", - XContentMapValues.extractValue("pivot_reviews.mappings._meta._data_frame.created_by", mappingAsMap)); + XContentMapValues.extractValue("pivot_reviews.mappings._meta.created_by", mappingAsMap)); } } diff --git a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndex.java b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndex.java index cbca203436aeb..6605269475c79 100644 --- a/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndex.java +++ b/x-pack/plugin/data-frame/src/main/java/org/elasticsearch/xpack/dataframe/persistence/DataframeIndex.java @@ -75,9 +75,9 @@ private static XContentBuilder createMappingXContent(Map mapping private static XContentBuilder addMetaData(XContentBuilder builder, String id) throws IOException { builder.startObject(META); + builder.field(DataFrameField.CREATED_BY, DataFrameField.DATA_FRAME_SIGNATURE); builder.startObject(DataFrameField.META_FIELDNAME); builder.field(DataFrameField.CREATION_DATE_MILLIS, System.currentTimeMillis()); - builder.field(DataFrameField.CREATED_BY, DataFrameField.DATA_FRAME_SIGNATURE); builder.startObject(DataFrameField.VERSION); builder.field(DataFrameField.CREATED, Version.CURRENT); builder.endObject();