From c679d0676dfff9182130ea0594de0e2b84a7da05 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 6 Mar 2018 10:43:14 -0800 Subject: [PATCH 1/2] Docs: Support triple quotes Adds support for triple quoted strings to the documentation test generator. Kibana's CONSOLE tool has supported them for a year but we were unable to use them in Elasticsearch's docs because the process that converts example snippets into tests couldn't handle this. This change adds code to convert them into standard JSON so we can pass them to Elasticsearch. --- .../doc/RestTestsFromSnippetsTask.groovy | 40 +++++++++++++++ .../doc/RestTestsFromSnippetsTaskTest.groovy | 50 +++++++++++++++++++ .../painless-getting-started.asciidoc | 47 ++++++++++++++--- docs/reference/ingest/ingest-node.asciidoc | 7 ++- 4 files changed, 135 insertions(+), 9 deletions(-) create mode 100644 buildSrc/src/test/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTaskTest.groovy diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy index 8491c5b45920e..95ec00beca7e0 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy @@ -19,6 +19,7 @@ package org.elasticsearch.gradle.doc +import groovy.transform.PackageScope import org.elasticsearch.gradle.doc.SnippetsTask.Snippet import org.gradle.api.InvalidUserDataException import org.gradle.api.tasks.Input @@ -99,6 +100,43 @@ public class RestTestsFromSnippetsTask extends SnippetsTask { return snippet.language == 'js' || snippet.curl } + /** + * Converts Kibana's block quoted strings into standard JSON. These + * {@code """} delimited strings can be embedded in CONSOLE and can + * contain newlines and {@code "} without the normal JSON escaping. + * This has to add it. + */ + @PackageScope + static String replaceBlockQuote(String body) { + int start = body.indexOf('"""'); + if (start < 0) { + return body + } + /* + * 1.3 is a fairly wild guess of the extra space needed to hold + * the escaped string. + */ + StringBuilder result = new StringBuilder((int) (body.length() * 1.3)); + int startOfNormal = 0; + while (start >= 0) { + int end = body.indexOf('"""', start + 3); + if (end < 0) { + throw new InvalidUserDataException( + "Invalid block quote starting at $start in:\n$body") + } + result.append(body.substring(startOfNormal, start)); + result.append('"'); + result.append(body.substring(start + 3, end) + .replace('"', '\\"') + .replace("\n", "\\n")); + result.append('"'); + startOfNormal = end + 3; + start = body.indexOf('"""', startOfNormal); + } + result.append(body.substring(startOfNormal)); + return result.toString(); + } + private class TestBuilder { private static final String SYNTAX = { String method = /(?GET|PUT|POST|HEAD|OPTIONS|DELETE)/ @@ -259,6 +297,8 @@ public class RestTestsFromSnippetsTask extends SnippetsTask { if (body != null) { // Throw out the leading newline we get from parsing the body body = body.substring(1) + // Replace """ quoted strings with valid json ones + body = replaceBlockQuote(body) current.println(" body: |") body.eachLine { current.println(" $it") } } diff --git a/buildSrc/src/test/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTaskTest.groovy b/buildSrc/src/test/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTaskTest.groovy new file mode 100644 index 0000000000000..0ec86263599f5 --- /dev/null +++ b/buildSrc/src/test/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTaskTest.groovy @@ -0,0 +1,50 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.doc + +import org.elasticsearch.gradle.doc.SnippetsTask.Snippet +import org.gradle.api.InvalidUserDataException + +import static org.elasticsearch.gradle.doc.RestTestsFromSnippetsTask.replaceBlockQuote + +class RestTestFromSnippetsTaskTest extends GroovyTestCase { + void testInvalidBlockQuote() { + String input = "\"foo\": \"\"\"bar\""; + String message = shouldFail({ replaceBlockQuote(input) }); + assertEquals("Invalid block quote starting at 7 in:\n$input", message); + } + + void testSimpleBlockQuote() { + assertEquals("\"foo\": \"bort baz\"", + replaceBlockQuote("\"foo\": \"\"\"bort baz\"\"\"")) + } + + void testMultipleBlockQuotes() { + assertEquals("\"foo\": \"bort baz\", \"bar\": \"other\"", + replaceBlockQuote("\"foo\": \"\"\"bort baz\"\"\", \"bar\": \"\"\"other\"\"\"")) + } + + void testEscapingInBlockQuote() { + assertEquals("\"foo\": \"bort\\\" baz\"", + replaceBlockQuote("\"foo\": \"\"\"bort\" baz\"\"\"")) + assertEquals("\"foo\": \"bort\\n baz\"", + replaceBlockQuote("\"foo\": \"\"\"bort\n baz\"\"\"")) + } +} diff --git a/docs/painless/painless-getting-started.asciidoc b/docs/painless/painless-getting-started.asciidoc index 7898631416b6b..e82e14b043840 100644 --- a/docs/painless/painless-getting-started.asciidoc +++ b/docs/painless/painless-getting-started.asciidoc @@ -53,7 +53,13 @@ GET hockey/_search "script_score": { "script": { "lang": "painless", - "source": "int total = 0; for (int i = 0; i < doc['goals'].length; ++i) { total += doc['goals'][i]; } return total;" + "source": """ + int total = 0; + for (int i = 0; i < doc['goals'].length; ++i) { + total += doc['goals'][i]; + } + return total; + """ } } } @@ -75,7 +81,13 @@ GET hockey/_search "total_goals": { "script": { "lang": "painless", - "source": "int total = 0; for (int i = 0; i < doc['goals'].length; ++i) { total += doc['goals'][i]; } return total;" + "source": """ + int total = 0; + for (int i = 0; i < doc['goals'].length; ++i) { + total += doc['goals'][i]; + } + return total; + """ } } } @@ -157,7 +169,10 @@ POST hockey/player/1/_update { "script": { "lang": "painless", - "source": "ctx._source.last = params.last; ctx._source.nick = params.nick", + "source": """ + ctx._source.last = params.last; + ctx._source.nick = params.nick + """, "params": { "last": "gaudreau", "nick": "hockey" @@ -228,7 +243,13 @@ POST hockey/player/_update_by_query { "script": { "lang": "painless", - "source": "if (ctx._source.last =~ /b/) {ctx._source.last += \"matched\"} else {ctx.op = 'noop'}" + "source": """ + if (ctx._source.last =~ /b/) { + ctx._source.last += "matched"; + } else { + ctx.op = "noop"; + } + """ } } ---------------------------------------------------------------- @@ -243,7 +264,13 @@ POST hockey/player/_update_by_query { "script": { "lang": "painless", - "source": "if (ctx._source.last ==~ /[^aeiou].*[aeiou]/) {ctx._source.last += \"matched\"} else {ctx.op = 'noop'}" + "source": """ + if (ctx._source.last ==~ /[^aeiou].*[aeiou]/) { + ctx._source.last += "matched"; + } else { + ctx.op = "noop"; + } + """ } } ---------------------------------------------------------------- @@ -296,7 +323,10 @@ POST hockey/player/_update_by_query { "script": { "lang": "painless", - "source": "ctx._source.last = ctx._source.last.replaceAll(/[aeiou]/, m -> m.group().toUpperCase(Locale.ROOT))" + "source": """ + ctx._source.last = ctx._source.last.replaceAll(/[aeiou]/, m -> + m.group().toUpperCase(Locale.ROOT)) + """ } } ---------------------------------------------------------------- @@ -311,7 +341,10 @@ POST hockey/player/_update_by_query { "script": { "lang": "painless", - "source": "ctx._source.last = ctx._source.last.replaceFirst(/[aeiou]/, m -> m.group().toUpperCase(Locale.ROOT))" + "source": """ + ctx._source.last = ctx._source.last.replaceFirst(/[aeiou]/, m -> + m.group().toUpperCase(Locale.ROOT)) + """ } } ---------------------------------------------------------------- diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc index 3c30648c701dc..7c626d6746f79 100644 --- a/docs/reference/ingest/ingest-node.asciidoc +++ b/docs/reference/ingest/ingest-node.asciidoc @@ -563,7 +563,7 @@ to set the index that the document will be indexed into: -------------------------------------------------- // NOTCONSOLE -Dynamic field names are also supported. This example sets the field named after the +Dynamic field names are also supported. This example sets the field named after the value of `service` to the value of the field `code`: [source,js] @@ -1829,7 +1829,10 @@ PUT _ingest/pipeline/my_index "processors": [ { "script": { - "source": " ctx._index = 'my_index'; ctx._type = '_doc' " + "source": """ + ctx._index = 'my_index'; + ctx._type = '_doc'; + """ } } ] From de4eb63656c07009de5f92220b8b41113fc4a181 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 6 Mar 2018 13:09:37 -0800 Subject: [PATCH 2/2] ; --- .../gradle/doc/RestTestsFromSnippetsTaskTest.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/test/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTaskTest.groovy b/buildSrc/src/test/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTaskTest.groovy index 0ec86263599f5..d0a7a2825e6f2 100644 --- a/buildSrc/src/test/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTaskTest.groovy +++ b/buildSrc/src/test/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTaskTest.groovy @@ -33,18 +33,18 @@ class RestTestFromSnippetsTaskTest extends GroovyTestCase { void testSimpleBlockQuote() { assertEquals("\"foo\": \"bort baz\"", - replaceBlockQuote("\"foo\": \"\"\"bort baz\"\"\"")) + replaceBlockQuote("\"foo\": \"\"\"bort baz\"\"\"")); } void testMultipleBlockQuotes() { assertEquals("\"foo\": \"bort baz\", \"bar\": \"other\"", - replaceBlockQuote("\"foo\": \"\"\"bort baz\"\"\", \"bar\": \"\"\"other\"\"\"")) + replaceBlockQuote("\"foo\": \"\"\"bort baz\"\"\", \"bar\": \"\"\"other\"\"\"")); } void testEscapingInBlockQuote() { assertEquals("\"foo\": \"bort\\\" baz\"", - replaceBlockQuote("\"foo\": \"\"\"bort\" baz\"\"\"")) + replaceBlockQuote("\"foo\": \"\"\"bort\" baz\"\"\"")); assertEquals("\"foo\": \"bort\\n baz\"", - replaceBlockQuote("\"foo\": \"\"\"bort\n baz\"\"\"")) + replaceBlockQuote("\"foo\": \"\"\"bort\n baz\"\"\"")); } }