From 25b452537f6c74e5f79f94fad9d918e6b3156757 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Mon, 24 Sep 2018 18:33:44 -0700 Subject: [PATCH 1/3] Painless: Add a reindex example to the context documentation. --- docs/painless/painless-contexts.asciidoc | 4 +- .../painless/painless-contexts/index.asciidoc | 4 +- .../painless-reindex-context.asciidoc | 182 +++++++++++++- .../painless/ContextExampleTests.java | 226 ++++++++++++++---- 4 files changed, 363 insertions(+), 53 deletions(-) diff --git a/docs/painless/painless-contexts.asciidoc b/docs/painless/painless-contexts.asciidoc index cc7bc752ec6d9..614811ee38d37 100644 --- a/docs/painless/painless-contexts.asciidoc +++ b/docs/painless/painless-contexts.asciidoc @@ -16,12 +16,12 @@ specialized code may define new ways to use a Painless script. | Elasticsearch Documentation | Ingest processor | <> | {ref}/script-processor.html[Elasticsearch Documentation] +| Reindex | <> + | {ref}/docs-reindex.html[Elasticsearch Documentation] | Update | <> | {ref}/docs-update.html[Elasticsearch Documentation] | Update by query | <> | {ref}/docs-update-by-query.html[Elasticsearch Documentation] -| Reindex | <> - | {ref}/docs-reindex.html[Elasticsearch Documentation] | Sort | <> | {ref}/search-request-sort.html[Elasticsearch Documentation] | Similarity | <> diff --git a/docs/painless/painless-contexts/index.asciidoc b/docs/painless/painless-contexts/index.asciidoc index a71fde0be32a0..9f92daf799d39 100644 --- a/docs/painless/painless-contexts/index.asciidoc +++ b/docs/painless/painless-contexts/index.asciidoc @@ -1,11 +1,11 @@ include::painless-ingest-processor-context.asciidoc[] +include::painless-reindex-context.asciidoc[] + include::painless-update-context.asciidoc[] include::painless-update-by-query-context.asciidoc[] -include::painless-reindex-context.asciidoc[] - include::painless-sort-context.asciidoc[] include::painless-similarity-context.asciidoc[] diff --git a/docs/painless/painless-contexts/painless-reindex-context.asciidoc b/docs/painless/painless-contexts/painless-reindex-context.asciidoc index ae5445183a6ad..971c8313de20f 100644 --- a/docs/painless/painless-contexts/painless-reindex-context.asciidoc +++ b/docs/painless/painless-contexts/painless-reindex-context.asciidoc @@ -65,4 +65,184 @@ reindexed into a target index. *API* -The standard <> is available. \ No newline at end of file +The standard <> is available. + +*Example* + +To run this example, first follow the steps in +<>. + +The example reindex script accomplishes the following: + +* Separates the `seats` index into two different new indexes - `afternoon` and + `evening` based on when a play begins. +* Removes the `date` and `time` fields that are extraneous since the + <> added a single combined + `datetime` field. +* Adds a `sold_datetime` field to each document that is filled in with the + current date and time of when a seat is sold. +* Sets the `sold` field to true along with a value for the `sold_datetime` field + for a specific subset of seats for use in future examples. + +[source,Painless] +---- +void setSold(Map _source) { <1> + if ((_source["number"] + _source["row"] % 4) % 3 == 0 || <2> + _source["number"] % 5 == 0) { + _source["sold"] = true; <3> + _source["sold_datetime"] = _source["datetime"] - <4> + 86400000 * (1 + _source["number"] % 14); + } +} + +void removeExtraneous(Map _source) { <5> + _source.remove("date"); <6> + _source.remove("time"); <7> +} + +setSold(ctx["_source"]); <8> +removeExtraneous(ctx["_source"]); <9> + +Instant instant = Instant.ofEpochMilli(ctx["_source"]["datetime"]); <10> +ZonedDateTime dt = instant.atZone(ZoneId.of("GMT+8")); <11> + +if (dt.getHour() > 16) { <12> + ctx["_index"] = "evening"; <13> +} +---- +<1> Creates a `setSold` <> to set a seat to sold + if it meets certain criteria for use in future examples. + Note:: + * The changes to the `Map` type value `_source` passed in to the function + are reflected throughout the script since it's a + <> value. +<2> Does a simple calculation to determine if the seat is sold based on + arbitrary criteria to create a set of sold seats for use in future examples. +<3> Sets the field `sold` to the boolean value `true`. +<4> Sets the field `sold_datetime` to a value between `1` to `15` days prior to + the play's date and time. + Note:: + The <> is used to guarantee + the number of days in the equation is evaluated prior to subtracting + the number of seconds from the play's date and time. +<5> Creates a `removeExtraneous` function to remove redundant fields from the + document. + Note:: + * The changes to the `Map` type value `_source` passed in to the function + are reflected throughout the script since it's a reference type value. +<6> Removes the `date` field from the document using the API non-static method + `remove`. +<7> Removes the `time` field from the document using the API non-static method + `remove`. +<8> Uses the `setSold` function to check if a seat is sold and update the + appropriate fields if so. + Note:: + * The use of the `ctx["_source"]` reindex context + <> to retrieve the `Map` reference type + value containing the document's fields. +<9> Uses the `removeExtraneous` function to remove redundant fields from the + document. + Note:: + * The use of the `ctx["_source"]` reindex context variable to retrieve the + `Map` reference type value containing the document's fields. +<10> Creates an `Instant` reference type variable `instant` and uses the API + static method `ofEpochMilli` to convert the `datetime` field from a + <> type value into an `Instant` reference type + value. + Note:: + * The use of the `ctx["_source"]` reindex context variable to retrieve the + `Map` reference type value containing the document's fields. +<11> Creates a `ZonedDateTime` reference type variable `dt` and uses the API + static method `atZone` to convert the `Instant` reference type value into a + `DateTime` reference type value. The time zone is specified using the + API static method `of` on the reference type `ZoneId`. +<12> Checks to see if the hour is past `4:00PM GMT+8 (PST)`. +<13> Changes from the default `afternoon` index (specified as part of the + upcoming curl request) to the `evening` index using the ctx["_index"] + reindex context variable. + +Submit the following requests: + +. Create {ref}/mapping.html[mappings] for the `afternoon` index: ++ +[source,js] +---- +PUT /afternoon +{ + "mappings": { + "seat": { + "properties": { + "theatre": { "type": "keyword" }, + "play": { "type": "text" }, + "actors": { "type": "text" }, + "row": { "type": "integer" }, + "number": { "type": "integer" }, + "cost": { "type": "double" }, + "sold": { "type": "boolean" }, + "sold_datetime": { "type": "date" }, + "datetime": { "type": "date" } + } + } + } +} +---- ++ +// CONSOLE + +. Create {ref}/mapping.html[mappings] for the `evening` index: ++ +[source,js] +---- +PUT /afternoon +{ + "mappings": { + "seat": { + "properties": { + "theatre": { "type": "keyword" }, + "play": { "type": "text" }, + "actors": { "type": "text" }, + "row": { "type": "integer" }, + "number": { "type": "integer" }, + "cost": { "type": "double" }, + "sold": { "type": "boolean" }, + "sold_datetime": { "type": "date" }, + "datetime": { "type": "date" } + } + } + } +} +---- ++ +// CONSOLE + +. Submit the reindex request. ++ +[source,js] +---- +POST reindex +{ + "source": { + "index": "seats" + }, + "dest": { + "index": "afternoon" + }, + "script": { + "source": "void setSold(Map _source) { if ((_source[\"number\"] + _source[\"row\"] % 4) % 3 == 0 || _source[\"number\"] % 5 == 0) { _source[\"sold\"] = true; _source[\"sold_datetime\"] = _source[\"datetime\"] - 86400000 * (1 + _source[\"number\"] % 14); } } void removeExtraneous(Map _source) { _source.remove(\"date\"); _source.remove(\"time\"); } setSold(ctx[\"_source\"]); removeExtraneous(ctx[\"_source\"]); Instant instant = Instant.ofEpochMilli(ctx[\"_source\"][\"datetime\"]); ZonedDateTime dt = instant.atZone(ZoneId.of(\"GMT+8\")); if (dt.getHour() > 16) { ctx[\"_index\"] = \"evening\"; }" + } +} +---- ++ +// CONSOLE + +. The reindex request may take some time to complete even after a successful + response is received. Submit the following request to check the number of + documents in each index. The `afternoon` index has 18312 documents, and the + `evening` index has 17892 documents. ++ +[source,js] +---- +GET /_cat/indices?v +---- ++ +// CONSOLE \ No newline at end of file diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ContextExampleTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ContextExampleTests.java index 15eed75bcb8df..bbce1efcf72e2 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ContextExampleTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ContextExampleTests.java @@ -19,6 +19,9 @@ package org.elasticsearch.painless; +import java.util.HashMap; +import java.util.Map; + /** * These tests run the Painless scripts used in the context docs against * slightly modified data designed around unit tests rather than a fully- @@ -26,7 +29,9 @@ */ public class ContextExampleTests extends ScriptTestCase { - // **** Docs Generator Code **** +// *** SETUP *** + + // Docs Generator Code: /* @@ -204,7 +209,7 @@ public static void main(String args[]) throws IOException { */ - // **** Initial Mappings **** + // Initial Mappings: /* @@ -231,6 +236,8 @@ public static void main(String args[]) throws IOException { */ +// *** EXAMPLE: INGEST *** + // Create Ingest to Modify Dates: /* @@ -252,55 +259,57 @@ public static void main(String args[]) throws IOException { public void testIngestProcessorScript() { assertEquals(1535785200000L, - exec("String[] split(String s, char d) {" + - " int count = 0;" + - " for (char c : s.toCharArray()) {" + - " if (c == d) {" + - " ++count;" + - " }" + - " }" + - " if (count == 0) {" + - " return new String[] {s};" + - " }" + - " String[] r = new String[count + 1];" + - " int i0 = 0, i1 = 0;" + - " count = 0;" + - " for (char c : s.toCharArray()) {" + - " if (c == d) {" + - " r[count++] = s.substring(i0, i1);" + - " i0 = i1 + 1;" + - " }" + - " ++i1;" + - " }" + - " r[count] = s.substring(i0, i1);" + - " return r;" + - "}" + - "def x = ['date': '2018-9-1', 'time': '3:00 PM'];" + - "String[] dateSplit = split(x.date, (char)'-');" + - "String year = dateSplit[0].trim();" + - "String month = dateSplit[1].trim();" + - "if (month.length() == 1) {" + - " month = '0' + month;" + - "}" + - "String day = dateSplit[2].trim();" + - "if (day.length() == 1) {" + - " day = '0' + day;" + - "}" + - "boolean pm = x.time.substring(x.time.length() - 2).equals('PM');" + - "String[] timeSplit = split(x.time.substring(0, x.time.length() - 2), (char)':');" + - "int hours = Integer.parseInt(timeSplit[0].trim());" + - "String minutes = timeSplit[1].trim();" + - "if (pm) {" + - " hours += 12;" + - "}" + - "String dts = year + '-' + month + '-' + day + " + - "'T' + (hours < 10 ? '0' + hours : '' + hours) + ':' + minutes + ':00+08:00';" + - "ZonedDateTime dt = ZonedDateTime.parse(dts, DateTimeFormatter.ISO_OFFSET_DATE_TIME);" + - "return dt.getLong(ChronoField.INSTANT_SECONDS) * 1000L" - ) + exec("String[] split(String s, char d) {" + + " int count = 0;" + + " for (char c : s.toCharArray()) {" + + " if (c == d) {" + + " ++count;" + + " }" + + " }" + + " if (count == 0) {" + + " return new String[] {s};" + + " }" + + " String[] r = new String[count + 1];" + + " int i0 = 0, i1 = 0;" + + " count = 0;" + + " for (char c : s.toCharArray()) {" + + " if (c == d) {" + + " r[count++] = s.substring(i0, i1);" + + " i0 = i1 + 1;" + + " }" + + " ++i1;" + + " }" + + " r[count] = s.substring(i0, i1);" + + " return r;" + + "}" + + "def x = ['date': '2018-9-1', 'time': '3:00 PM'];" + + "String[] dateSplit = split(x.date, (char)'-');" + + "String year = dateSplit[0].trim();" + + "String month = dateSplit[1].trim();" + + "if (month.length() == 1) {" + + " month = '0' + month;" + + "}" + + "String day = dateSplit[2].trim();" + + "if (day.length() == 1) {" + + " day = '0' + day;" + + "}" + + "boolean pm = x.time.substring(x.time.length() - 2).equals('PM');" + + "String[] timeSplit = split(x.time.substring(0, x.time.length() - 2), (char)':');" + + "int hours = Integer.parseInt(timeSplit[0].trim());" + + "String minutes = timeSplit[1].trim();" + + "if (pm) {" + + " hours += 12;" + + "}" + + "String dts = year + '-' + month + '-' + day + " + + "'T' + (hours < 10 ? '0' + hours : '' + hours) + ':' + minutes + ':00+08:00';" + + "ZonedDateTime dt = ZonedDateTime.parse(dts, DateTimeFormatter.ISO_OFFSET_DATE_TIME);" + + "return dt.getLong(ChronoField.INSTANT_SECONDS) * 1000L" + ) ); } +// *** UPLOAD *** + // Post Generated Data: /* @@ -308,4 +317,125 @@ public void testIngestProcessorScript() { curl -XPOST localhost:9200/seats/seat/_bulk?pipeline=seats -H "Content-Type: application/x-ndjson" --data-binary "@/home/jdconrad/test/seats.json" */ + +// *** EXAMPLE: REINDEX *** + + // Create Mappings for Reindex: + + /* + + curl -X PUT "localhost:9200/afternoon" -H 'Content-Type: application/json' -d' + { + "mappings": { + "seat": { + "properties": { + "theatre": { "type": "keyword" }, + "play": { "type": "text" }, + "actors": { "type": "text" }, + "row": { "type": "integer" }, + "number": { "type": "integer" }, + "cost": { "type": "double" }, + "sold": { "type": "boolean" }, + "sold_datetime": { "type": "date" }, + "datetime": { "type": "date" } + } + } + } + } + ' + + */ + + /* + + curl -X PUT "localhost:9200/evening" -H 'Content-Type: application/json' -d' + { + "mappings": { + "seat": { + "properties": { + "theatre": { "type": "keyword" }, + "play": { "type": "text" }, + "actors": { "type": "text" }, + "row": { "type": "integer" }, + "number": { "type": "integer" }, + "cost": { "type": "double" }, + "sold": { "type": "boolean" }, + "sold_datetime": { "type": "date" }, + "datetime": { "type": "date" } + } + } + } + } + ' + + */ + + // Use the reindex script to create afternoon and evening indexes + + /* + + curl -X POST "localhost:9200/_reindex" -H 'Content-Type: application/json' -d' + { + "source": { + "index": "seats" + }, + "dest": { + "index": "afternoon" + }, + "script": { + "source": "void setSold(Map _source) { if ((_source[\"number\"] + _source[\"row\"] % 4) % 3 == 0 || _source[\"number\"] % 5 == 0) { _source[\"sold\"] = true; _source[\"sold_datetime\"] = _source[\"datetime\"] - 86400000 * (1 + _source[\"number\"] % 14); } } void removeExtraneous(Map _source) { _source.remove(\"date\"); _source.remove(\"time\"); } setSold(ctx[\"_source\"]); removeExtraneous(ctx[\"_source\"]); Instant instant = Instant.ofEpochMilli(ctx[\"_source\"][\"datetime\"]); ZonedDateTime dt = instant.atZone(ZoneId.of(\"GMT+8\")); if (dt.getHour() > 16) { ctx[\"_index\"] = \"evening\"; }" + } + } + ' + + */ + + public void testReindexScript() { + Map _source = new HashMap(); + _source.put("number", 10); + _source.put("row", 1); + _source.put("sold", true); + _source.put("datetime", 1522573200000L); + _source.put("sold_datetime", 1521622800000L); + Map ctx = new HashMap<>(); + ctx.put("_source", _source); + ctx.put("_index", "evening"); + + assertEquals(ctx, exec( + "void setSold(Map _source) {" + + " if ((_source[\"number\"] + _source[\"row\"] % 4) % 3 == 0 || _source[\"number\"] % 5 == 0) {" + + " _source[\"sold\"] = true;" + + " _source[\"sold_datetime\"] = _source[\"datetime\"] - 86400000 * (1 + _source[\"number\"] % 14);" + + " }" + + "}" + + "" + + "void removeExtraneous(Map _source) {" + + " _source.remove(\"date\");" + + " _source.remove(\"time\");" + + "}" + + "" + + "Map _source = new HashMap();" + + "_source[\"number\"] = 10;" + + "_source[\"row\"] = 1;" + + "_source[\"sold\"] = false;" + + "_source[\"datetime\"] = 1522573200000L;" + + "_source[\"date\"] = \"2018-08-17\";" + + "_source[\"time\"] = \"12:30PM\";" + + "Map rtn = new HashMap();" + + "rtn[\"_source\"] = _source;" + + "" + + "setSold(rtn[\"_source\"]);" + + "removeExtraneous(rtn[\"_source\"]);" + + "" + + "Instant instant = Instant.ofEpochMilli(rtn[\"_source\"][\"datetime\"]);" + + "ZonedDateTime dt = instant.atZone(ZoneId.of(\"GMT+8\"));" + + "" + + "if (dt.getHour() > 16) {" + + " rtn[\"_index\"] = \"evening\";" + + "}" + + "" + + "return rtn;" + ) + ); + } } From e876161aca4604733d7fc1efb4f995e87d8935d7 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Mon, 24 Sep 2018 19:01:50 -0700 Subject: [PATCH 2/3] Fix console error. --- .../painless-reindex-context.asciidoc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/painless/painless-contexts/painless-reindex-context.asciidoc b/docs/painless/painless-contexts/painless-reindex-context.asciidoc index 971c8313de20f..9666a7360fe68 100644 --- a/docs/painless/painless-contexts/painless-reindex-context.asciidoc +++ b/docs/painless/painless-contexts/painless-reindex-context.asciidoc @@ -193,7 +193,7 @@ PUT /afternoon + [source,js] ---- -PUT /afternoon +PUT /evening { "mappings": { "seat": { @@ -215,11 +215,11 @@ PUT /afternoon + // CONSOLE -. Submit the reindex request. +. Submit the reindex request: + [source,js] ---- -POST reindex +POST /_reindex { "source": { "index": "seats" @@ -234,11 +234,12 @@ POST reindex ---- + // CONSOLE +// TEST[skip: requires setup from other pages] . The reindex request may take some time to complete even after a successful - response is received. Submit the following request to check the number of - documents in each index. The `afternoon` index has 18312 documents, and the - `evening` index has 17892 documents. + response is received. The `afternoon` index has 18312 documents, and the + `evening` index has 17892 documents. Submit the following request to check the + number of documents in each index: + [source,js] ---- From 368aaf67f9c60a4bcdfcf33038ad6c544368e194 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Tue, 25 Sep 2018 07:53:36 -0700 Subject: [PATCH 3/3] Fix test. --- .../java/org/elasticsearch/painless/ContextExampleTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ContextExampleTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ContextExampleTests.java index bbce1efcf72e2..e076e897afb71 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ContextExampleTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ContextExampleTests.java @@ -391,7 +391,7 @@ public void testIngestProcessorScript() { */ public void testReindexScript() { - Map _source = new HashMap(); + Map _source = new HashMap<>(); _source.put("number", 10); _source.put("row", 1); _source.put("sold", true);